scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-10 21:07:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'upstream/master'

Browse source
This commit is contained in:
Todd Gruben 2018-05-07 09:22:30 -05:00
commit 2c9d238acf
84 changed files with 4617 additions and 3121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.travis.yml
View file

@ -7,6 +7,9 @@ env:
global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
- secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA="
- secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o="
matrix:
- GOARCH=386
- GOARCH=amd64
install:
- make install-dep install-statik vendor generate-statik
script:
@ -24,6 +27,7 @@ deploy:
on:
all_branches: true
go: "1.10"
condition: $GOARCH = amd64
matrix:
allow_failures:
- go: master

33
CHANGELOG.md
View file

@ -5,7 +5,11 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
## Unreleased
## [v0.9.0] - 2018-05-04
This version contains 188 contribution from 12 contributors. There are 141 files changed, 17,832 insertions, and 7,503 deletions.
*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.*
### Added
@ -19,11 +23,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064))
- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063))
- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048))
- Adds support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014))
- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013))
- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014))
- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245))
- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985))
- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963))
- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972))
- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191))
- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246))
- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250))
### Changed
@ -39,14 +45,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027))
- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001))
- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950))
- Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991))
- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991))
- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907))
- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220))
- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223))
- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216))
- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221))
### Removed
- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171))
- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142))
- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130))
- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226))
- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222))
- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219))
- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212))
- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209))
- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207))
### Fixed
@ -59,6 +75,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084))
- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979))
- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980))
- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233))
- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229))
- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228))
### Deprecated
- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205))
### Performance

4
Dockerfile
View file

@ -1,4 +1,4 @@
FROM golang:1.10 as builder
FROM golang:1.10.2 as builder
COPY . /go/src/github.com/pilosa/pilosa/
@ -11,6 +11,8 @@ LABEL maintainer "dev@pilosa.com"
COPY --from=builder /go/bin/pilosa /pilosa
COPY NOTICE /NOTICE
EXPOSE 10101
VOLUME /data

58
Gopkg.lock generated
View file

@ -57,7 +57,10 @@
[[projects]]
name = "github.com/go-ole/go-ole"
packages = [".","oleutil"]
packages = [
".",
"oleutil"
]
revision = "0e87ea779d9deb219633b828a023b32e1244dd57"
version = "v1.2.0"
@ -67,12 +70,6 @@
revision = "100ba4e885062801d56799d78530b73b178a78f3"
version = "v0.4"
[[projects]]
branch = "master"
name = "github.com/golang/groupcache"
packages = ["lru"]
revision = "84a468cf14b4376def5d68c722b139b881c450a4"
[[projects]]
branch = "master"
name = "github.com/golang/protobuf"
@ -130,7 +127,17 @@
[[projects]]
branch = "master"
name = "github.com/hashicorp/hcl"
packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"]
packages = [
".",
"hcl/ast",
"hcl/parser",
"hcl/scanner",
"hcl/strconv",
"hcl/token",
"json/parser",
"json/scanner",
"json/token"
]
revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8"
[[projects]]
@ -154,7 +161,10 @@
[[projects]]
branch = "master"
name = "github.com/miekg/dns"
packages = [".","internal/socket"]
packages = [
".",
"internal/socket"
]
revision = "9fc4eb252eedf0ef8adc05169ce35da5e31beaba"
[[projects]]
@ -195,7 +205,14 @@
[[projects]]
name = "github.com/shirou/gopsutil"
packages = ["cpu","host","internal/common","mem","net","process"]
packages = [
"cpu",
"host",
"internal/common",
"mem",
"net",
"process"
]
revision = "bfe3c2e8f406bf352bc8df81f98c752224867349"
version = "v2.17.11"
@ -208,7 +225,10 @@
[[projects]]
branch = "master"
name = "github.com/spf13/afero"
packages = [".","mem"]
packages = [
".",
"mem"
]
revision = "5660eeed305fe5f69c8fc6cf899132a459a97064"
[[projects]]
@ -256,13 +276,23 @@
[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = ["unix","windows"]
packages = [
"unix",
"windows"
]
revision = "1e2299c37cc91a509f1b12369872d27be0ce98a6"
[[projects]]
branch = "master"
name = "golang.org/x/text"
packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"]
packages = [
"internal/gen",
"internal/triegen",
"internal/ucd",
"transform",
"unicode/cldr",
"unicode/norm"
]
revision = "88f656faf3f37f690df1a32515b479415e1a6769"
[[projects]]
@ -274,6 +304,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "668c03e22d947dd7e24c4c5e07d7850ed0a2ea9a27d2af1fa0d1cff0e3743b0b"
inputs-digest = "61a00007fe7398d958fc8668543d42293f1310f9f9b74c63ff96f5a892eab1e7"
solver-name = "gps-cdcl"
solver-version = 1

8
Makefile
View file

@ -42,7 +42,7 @@ build: vendor
# Create a single release build under the build directory
release-build: vendor
$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa"
cp LICENSE README.md build/pilosa-$(VERSION_ID)
cp NOTICE LICENSE README.md build/pilosa-$(VERSION_ID)
tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
@ -80,8 +80,12 @@ generate-protoc: require-protoc require-protoc-gen-gofast
generate-statik: require-statik
go generate github.com/pilosa/pilosa/statik
# `go generate` stringers
generate-stringer:
go generate github.com/pilosa/pilosa
# `go generate` all needed packages
generate: generate-protoc generate-statik
generate: generate-protoc generate-statik generate-stringer
# Create Docker image from Dockerfile
docker:

36
NOTICE Normal file
View file

@ -0,0 +1,36 @@
Software license
================
Copyright 2017 Pilosa Corp.
Licensed under the Apache License, Version 2.0 (the "License").
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Third-party software licenses
=============================
The file /pilosa/lru/lru.go contains a redistribution of lru
(github.com/golang/groupcache/lru); the license follows:
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

1205
api.go Normal file
View file

File diff suppressed because it is too large Load diff

16
apimethod_string.go Normal file
View file

@ -0,0 +1,16 @@
// Code generated by "stringer -type=apiMethod"; DO NOT EDIT.
package pilosa
import "fmt"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 327, 347, 360, 374, 389, 406, 419, 439, 447, 460}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
return fmt.Sprintf("apiMethod(%d)", i)
}
return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]]
}

13
broadcast_test.go
View file

@ -15,12 +15,16 @@
package pilosa_test
import (
"bytes"
"reflect"
"testing"
"io/ioutil"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/server"
)
// Ensure a message can be marshaled and unmarshaled.
@ -52,8 +56,13 @@ func testMessageMarshal(t *testing.T, m proto.Message) {
// Ensure that BroadcastReceiver can register a BroadcastHandler.
func TestBroadcast_BroadcastReceiver(t *testing.T) {
s := pilosa.NewServer()
com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard)
com.Config.Bind = "localhost:0"
err := com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO
if err != nil {
t.Fatalf("setting up server: %v", err)
}
s := com.Server
sbr := NewSimpleBroadcastReceiver()
sbh := NewSimpleBroadcastHandler()

2
cache.go
View file

@ -22,8 +22,8 @@ import (
"sync"
"time"
"github.com/golang/groupcache/lru"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/lru"
)
const (

108
client_test.go
View file

@ -26,6 +26,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
@ -36,10 +37,10 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
for i := 0; i < numNodes; i++ {
hldr[i] = test.MustOpenHolder()
server[i] = test.NewServer()
server[i].Handler.Cluster = c
server[i].Handler.Cluster.Nodes[i].URI = server[i].HostURI()
server[i].Handler.Holder = hldr[i].Holder
server[i].Handler.Node = server[i].Handler.Cluster.Nodes[i]
server[i].Handler.API.URI = server[i].HostURI()
server[i].Handler.API.Cluster = c
server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI()
server[i].Handler.API.Holder = hldr[i].Holder
}
return server, hldr
}
@ -47,7 +48,7 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
var defaultClient *http.Client
func init() {
defaultClient = pilosa.GetHTTPClient(nil)
defaultClient = server.GetHTTPClient(nil)
}
@ -86,7 +87,7 @@ func TestClient_MultiNode(t *testing.T) {
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
sliceNums := []uint64{1, 2, 6}
for i, num := range sliceNums {
owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].HostURI())
owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())
ownsNum := false
for _, ownNum := range owns {
if ownNum == num {
@ -217,10 +218,10 @@ func TestClient_Import(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -268,10 +269,10 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -310,17 +311,17 @@ func TestClient_ImportValue(t *testing.T) {
// Load bitmap into cache to ensure cache gets updated.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true, Fields: []*pilosa.Field{&fld}})
frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}})
if err != nil {
t.Fatal(err)
}
s := test.NewServer()
defer s.Close()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -332,14 +333,44 @@ func TestClient_ImportValue(t *testing.T) {
t.Fatal(err)
}
// Verify Sum.
sum, cnt, err := frame.FieldSum(nil, fld.Name)
if err != nil {
t.Fatal(err)
}
// Verify data.
if sum != 50 || cnt != 3 {
t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=70, cnt=3", sum, cnt)
t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt)
}
// Verify Min.
min, cnt, err := frame.FieldMin(nil, fld.Name)
if err != nil {
t.Fatal(err)
}
if min != -10 || cnt != 1 {
t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt)
}
// Verify Min with Filter.
filter, err := frame.FieldRange(fld.Name, pql.GT, 40)
if err != nil {
t.Fatal(err)
}
min, cnt, err = frame.FieldMin(filter, fld.Name)
if err != nil {
t.Fatal(err)
}
if min != -100 || cnt != 0 {
t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt)
}
// Verify Max.
max, cnt, err := frame.FieldMax(nil, fld.Name)
if err != nil {
t.Fatal(err)
}
if max != 40 || cnt != 1 {
t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt)
}
}
@ -355,10 +386,10 @@ func TestClient_BackupRestore(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
@ -420,10 +451,11 @@ func TestClient_BackupInverseView(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
@ -457,10 +489,10 @@ func TestClient_BackupInvalidView(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
@ -486,10 +518,10 @@ func TestClient_FragmentBlocks(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
s.Handler.API.URI = s.HostURI()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Retrieve blocks.
c := test.MustNewClient(s.Host(), defaultClient)

92
cluster.go
View file

@ -17,7 +17,6 @@ package pilosa
import (
"context"
"encoding/binary"
"errors"
"fmt"
"hash/fnv"
"io/ioutil"
@ -33,6 +32,7 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
)
@ -40,9 +40,6 @@ const (
// DefaultPartitionN is the default number of partitions in a cluster.
DefaultPartitionN = 256
// DefaultReplicaN is the default number of replicas per partition.
DefaultReplicaN = 1
// ClusterState represents the state returned in the /status endpoint.
ClusterStateStarting = "STARTING"
ClusterStateNormal = "NORMAL"
@ -264,7 +261,6 @@ type Cluster struct {
// Close management
wg sync.WaitGroup
closing chan struct{}
prefect SecurityManager
Logger Logger
@ -275,19 +271,17 @@ type Cluster struct {
// NewCluster returns a new instance of Cluster with defaults.
func NewCluster() *Cluster {
return &Cluster{
Hasher: &jmphasher{},
PartitionN: DefaultPartitionN,
ReplicaN: DefaultReplicaN,
MaxWritesPerRequest: DefaultMaxWritesPerRequest,
EventReceiver: NopEventReceiver,
Hasher: &jmphasher{},
PartitionN: DefaultPartitionN,
ReplicaN: 1,
EventReceiver: NopEventReceiver,
joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel
jobs: make(map[int64]*ResizeJob),
closing: make(chan struct{}),
joining: make(chan struct{}),
Logger: NopLogger,
prefect: &NopSecurityManager{},
Logger: NopLogger,
}
}
@ -298,6 +292,12 @@ func (c *Cluster) CoordinatorNode() *Node {
// IsCoordinator is true if this node is the coordinator.
func (c *Cluster) IsCoordinator() bool {
c.mu.RLock()
defer c.mu.RUnlock()
return c.isCoordinator()
}
func (c *Cluster) isCoordinator() bool {
return c.Coordinator == c.Node.ID
}
@ -306,6 +306,8 @@ func (c *Cluster) IsCoordinator() bool {
// will consider itself coordinator and update the other
// nodes with its version of Cluster.Status.
func (c *Cluster) SetCoordinator(n *Node) error {
c.mu.Lock()
defer c.mu.Unlock()
// Verify that the new Coordinator value matches
// this node.
if c.Node.ID != n.ID {
@ -313,7 +315,7 @@ func (c *Cluster) SetCoordinator(n *Node) error {
}
// Update IsCoordinator on all nodes (locally).
_ = c.UpdateCoordinator(n)
_ = c.updateCoordinator(n)
// Send the update coordinator message to all nodes.
err := c.Broadcaster.SendSync(
@ -333,6 +335,12 @@ func (c *Cluster) SetCoordinator(n *Node) error {
// to true, and sets all other nodes to false. Returns true if the value
// changed.
func (c *Cluster) UpdateCoordinator(n *Node) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.updateCoordinator(n)
}
func (c *Cluster) updateCoordinator(n *Node) bool {
var changed bool
if c.Coordinator != n.ID {
c.Coordinator = n.ID
@ -434,19 +442,11 @@ func (c *Cluster) setState(state string) {
var doCleanup bool
switch state {
case ClusterStateResizing:
c.prefect.SetRestricted()
case ClusterStateNormal:
c.prefect.SetNormal()
// Don't change routing for these states:
// - ClusterStateStarting
// If state is RESIZING -> NORMAL then run cleanup.
if c.state == ClusterStateResizing {
doCleanup = true
}
default:
panic(fmt.Sprintf("invalid cluster state: %s", state))
}
c.state = state
@ -525,6 +525,12 @@ func (c *Cluster) Status() *internal.ClusterStatus {
}
}
func (c *Cluster) NodeByID(id string) *Node {
c.mu.RLock()
defer c.mu.RUnlock()
return c.nodeByID(id)
}
// nodeByID returns a node reference by ID.
func (c *Cluster) nodeByID(id string) *Node {
for _, n := range c.Nodes {
@ -654,7 +660,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost {
func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFrame) fragsByHost {
t := make(fragsByHost)
for i := uint64(0); i <= maxSlice; i++ {
nodes := c.FragmentNodes(idx, i)
nodes := c.SliceNodes(idx, i)
for _, n := range nodes {
// for each frame/view combination:
for frame, views := range frameViews {
@ -807,14 +813,14 @@ func (c *Cluster) Partition(index string, slice uint64) int {
return int(h.Sum64() % uint64(c.PartitionN))
}
// FragmentNodes returns a list of nodes that own a fragment.
func (c *Cluster) FragmentNodes(index string, slice uint64) []*Node {
// SliceNodes returns a list of nodes that own a fragment.
func (c *Cluster) SliceNodes(index string, slice uint64) []*Node {
return c.PartitionNodes(c.Partition(index, slice))
}
// OwnsFragment returns true if a host owns a fragment.
func (c *Cluster) OwnsFragment(nodeID string, index string, slice uint64) bool {
return Nodes(c.FragmentNodes(index, slice)).ContainsID(nodeID)
// OwnsSlice returns true if a host owns a fragment.
func (c *Cluster) OwnsSlice(nodeID string, index string, slice uint64) bool {
return Nodes(c.SliceNodes(index, slice)).ContainsID(nodeID)
}
// PartitionNodes returns a list of nodes that own a partition.
@ -913,7 +919,10 @@ func (c *Cluster) Open() error {
}
// Add the local node to the cluster.
c.AddNode(c.Node)
err := c.AddNode(c.Node)
if err != nil {
return errors.Wrap(err, "adding local node")
}
// Start the EventReceiver.
if err := c.EventReceiver.Start(c); err != nil {
@ -1197,8 +1206,11 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob,
func (c *Cluster) CompleteCurrentJob(state string) error {
c.mu.Lock()
defer c.mu.Unlock()
if !c.isCoordinator() {
return ErrNodeNotCoordinator
}
if c.currentJob == nil {
return fmt.Errorf("no resize job currently running")
return ErrResizeNotRunning
}
c.currentJob.SetState(state)
c.currentJob = nil
@ -1698,13 +1710,15 @@ func (c *Cluster) nodeJoin(node *Node) error {
// Only change to normal if there is no existing data. Otherwise,
// the coordinator needs to wait to receive READY messages (nodeStates)
// from remote nodes before setting the cluster to state NORMAL.
if !c.Holder.HasData() {
if ok, err := c.Holder.HasData(); !ok && err == nil {
// If the result of the previous AddNode completed the joining of nodes
// in the topology, then change the state to NORMAL.
if c.haveTopologyAgreement() {
return c.setStateAndBroadcast(ClusterStateNormal)
}
return nil
} else if err != nil {
return errors.Wrap(err, "checking if holder has data")
}
if c.haveTopologyAgreement() && c.allNodesReady() {
@ -1726,11 +1740,13 @@ func (c *Cluster) nodeJoin(node *Node) error {
}
// If the holder does not yet contain data, go ahead and add the node.
if !c.Holder.HasData() {
if ok, err := c.Holder.HasData(); !ok && err == nil {
if err := c.AddNode(node); err != nil {
return err
}
return c.setStateAndBroadcast(ClusterStateNormal)
} else if err != nil {
return errors.Wrap(err, "checking if holder has data2")
}
// If the cluster has data, we need to change to RESIZING and
@ -1747,7 +1763,7 @@ func (c *Cluster) nodeJoin(node *Node) error {
func (c *Cluster) NodeLeave(node *Node) error {
// Refuse the request if this is not the coordinator.
if !c.IsCoordinator() {
return fmt.Errorf("Node removal requests are only valid on the Coordinator node: %s", c.CoordinatorNode().ID)
return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.CoordinatorNode().ID)
}
if c.State() != ClusterStateNormal {
@ -1761,7 +1777,7 @@ func (c *Cluster) NodeLeave(node *Node) error {
// Prevent removing the coordinator node (this node).
if node.ID == c.Node.ID {
return fmt.Errorf("The coordinator node cannot be removed. First, make a different node the new coordinator.")
return fmt.Errorf("coordinator cannot be removed; first, make a different node the new coordinator.")
}
// See if resize job can be generated
@ -1784,11 +1800,13 @@ func (c *Cluster) nodeLeave(node *Node) error {
}
// If the holder does not yet contain data, go ahead and remove the node.
if !c.Holder.HasData() {
if ok, err := c.Holder.HasData(); !ok && err == nil {
if err := c.RemoveNode(n); err != nil {
return err
}
return c.setStateAndBroadcast(ClusterStateNormal)
} else if err != nil {
return errors.Wrap(err, "checking if holder has data")
}
// If the cluster has data then change state to RESIZING and
@ -1802,9 +1820,11 @@ func (c *Cluster) nodeLeave(node *Node) error {
}
func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error {
c.mu.Lock()
defer c.mu.Unlock()
c.Logger.Printf("merge cluster status: %v", cs)
// Ignore status updates from self (coordinator).
if c.IsCoordinator() {
if c.isCoordinator() {
return nil
}
@ -1841,7 +1861,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error {
}
}
c.SetState(cs.State)
c.setState(cs.State)
c.markAsJoined()

171
cluster_test.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
package pilosa
import (
"bytes"
@ -22,29 +22,27 @@ import (
"testing/quick"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := pilosa.Cluster{
Nodes: []*pilosa.Node{
{URI: test.NewURIFromHostPort("serverA", 1000)},
{URI: test.NewURIFromHostPort("serverB", 1000)},
{URI: test.NewURIFromHostPort("serverC", 1000)},
c := Cluster{
Nodes: []*Node{
{URI: NewTestURIFromHostPort("serverA", 1000)},
{URI: NewTestURIFromHostPort("serverB", 1000)},
{URI: NewTestURIFromHostPort("serverC", 1000)},
},
Hasher: test.NewModHasher(),
Hasher: NewTestModHasher(),
ReplicaN: 2,
}
// Verify nodes are distributed.
if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[0], c.Nodes[1]}) {
if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[2], c.Nodes[0]}) {
if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
@ -52,7 +50,7 @@ func TestCluster_Owners(t *testing.T) {
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
c := pilosa.NewCluster()
c := NewCluster()
c.PartitionN = partitionN
partitionID := c.Partition(index, slice)
@ -85,7 +83,7 @@ func TestHasher(t *testing.T) {
{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
} {
for i, v := range tt.bucket {
if got := pilosa.NewHasher().Hash(tt.key, i+1); got != v {
if got := NewHasher().Hash(tt.key, i+1); got != v {
t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
}
}
@ -94,8 +92,8 @@ func TestHasher(t *testing.T) {
// Ensure OwnsSlices can find the actual slice list for node and index.
func TestCluster_OwnsSlices(t *testing.T) {
c := test.NewCluster(5)
slices := c.OwnsSlices("test", 10, test.NewURIFromHostPort("host2", 0))
c := NewTestCluster(5)
slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0))
if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
@ -104,7 +102,7 @@ func TestCluster_OwnsSlices(t *testing.T) {
// Ensure ContainsSlices can find the actual slice list for node and index.
func TestCluster_ContainsSlices(t *testing.T) {
c := test.NewCluster(5)
c := NewTestCluster(5)
c.ReplicaN = 3
slices := c.ContainsSlices("test", 10, c.Nodes[2])
@ -114,20 +112,20 @@ func TestCluster_ContainsSlices(t *testing.T) {
}
func TestCluster_Nodes(t *testing.T) {
uri0 := test.NewURIFromHostPort("node0", 0)
uri1 := test.NewURIFromHostPort("node1", 0)
uri2 := test.NewURIFromHostPort("node2", 0)
uri3 := test.NewURIFromHostPort("node3", 0)
uri0 := NewTestURIFromHostPort("node0", 0)
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
uri3 := NewTestURIFromHostPort("node3", 0)
node0 := &pilosa.Node{ID: "node0", URI: uri0}
node1 := &pilosa.Node{ID: "node1", URI: uri1}
node2 := &pilosa.Node{ID: "node2", URI: uri2}
node3 := &pilosa.Node{ID: "node3", URI: uri3}
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
node3 := &Node{ID: "node3", URI: uri3}
nodes := []*pilosa.Node{node0, node1, node2}
nodes := []*Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := pilosa.Nodes(nodes).IDs()
actual := Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
@ -135,24 +133,24 @@ func TestCluster_Nodes(t *testing.T) {
})
t.Run("Filter", func(t *testing.T) {
actual := pilosa.Nodes(pilosa.Nodes(nodes).Filter(nodes[1])).URIs()
expected := []pilosa.URI{uri0, uri2}
actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := pilosa.Nodes(pilosa.Nodes(nodes).FilterURI(uri1)).URIs()
expected := []pilosa.URI{uri0, uri2}
actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := pilosa.Nodes(nodes).Contains(node1)
actualFalse := pilosa.Nodes(nodes).Contains(node3)
actualTrue := Nodes(nodes).Contains(node1)
actualFalse := Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
@ -162,9 +160,9 @@ func TestCluster_Nodes(t *testing.T) {
})
t.Run("Clone", func(t *testing.T) {
clone := pilosa.Nodes(nodes).Clone()
actual := pilosa.Nodes(clone).URIs()
expected := []pilosa.URI{uri0, uri1, uri2}
clone := Nodes(nodes).Clone()
actual := Nodes(clone).URIs()
expected := []URI{uri0, uri1, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
@ -172,16 +170,16 @@ func TestCluster_Nodes(t *testing.T) {
}
func TestCluster_Coordinator(t *testing.T) {
uri1 := test.NewURIFromHostPort("node1", 0)
uri2 := test.NewURIFromHostPort("node2", 0)
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
node1 := &pilosa.Node{ID: "node1", URI: uri1}
node2 := &pilosa.Node{ID: "node2", URI: uri2}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
c1 := *pilosa.NewCluster()
c1 := *NewCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *pilosa.NewCluster()
c2 := *NewCluster()
c2.Node = node2
c2.Coordinator = node1.ID
@ -195,17 +193,17 @@ func TestCluster_Coordinator(t *testing.T) {
}
func TestCluster_Topology(t *testing.T) {
c1 := test.NewCluster(1) // automatically creates Node{ID: "node0"}
c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
uri0 := test.NewURIFromHostPort("host0", 0)
uri1 := test.NewURIFromHostPort("host1", 0)
uri2 := test.NewURIFromHostPort("host2", 0)
invalid := test.NewURIFromHostPort("invalid", 0)
uri0 := NewTestURIFromHostPort("host0", 0)
uri1 := NewTestURIFromHostPort("host1", 0)
uri2 := NewTestURIFromHostPort("host2", 0)
invalid := NewTestURIFromHostPort("invalid", 0)
node0 := &pilosa.Node{ID: "node0", URI: uri0}
node1 := &pilosa.Node{ID: "node1", URI: uri1}
node2 := &pilosa.Node{ID: "node2", URI: uri2}
nodeinvalid := &pilosa.Node{ID: "nodeinvalid", URI: invalid}
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.AddNode(node1)
@ -243,7 +241,7 @@ func TestCluster_Topology(t *testing.T) {
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := test.NewTestCluster(1)
tc := NewClusterCluster(1)
// Open TestCluster.
if err := tc.Open(); err != nil {
@ -253,11 +251,11 @@ func TestCluster_ResizeStates(t *testing.T) {
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != pilosa.ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State())
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
expectedTop := &pilosa.Topology{
expectedTop := &Topology{
NodeIDs: []string{node.Node.ID},
}
@ -273,13 +271,13 @@ func TestCluster_ResizeStates(t *testing.T) {
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &pilosa.Topology{
top := &Topology{
NodeIDs: []string{node.Node.ID},
}
tc.WriteTopology(node.Path, top)
@ -290,8 +288,8 @@ func TestCluster_ResizeStates(t *testing.T) {
}
// Ensure that node comes up in state NORMAL.
if node.State() != pilosa.ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State())
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
// Close TestCluster.
@ -301,13 +299,13 @@ func TestCluster_ResizeStates(t *testing.T) {
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &pilosa.Topology{
top := &Topology{
NodeIDs: []string{"some-other-host"},
}
tc.WriteTopology(node.Path, top)
@ -326,7 +324,7 @@ func TestCluster_ResizeStates(t *testing.T) {
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc := NewClusterCluster(0)
tc.AddNode(false)
// Open TestCluster.
@ -340,13 +338,13 @@ func TestCluster_ResizeStates(t *testing.T) {
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State())
} else if node1.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State())
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &pilosa.Topology{
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
@ -364,12 +362,12 @@ func TestCluster_ResizeStates(t *testing.T) {
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// write topology to data file
top := &pilosa.Topology{
top := &Topology{
NodeIDs: []string{"node0", "node2"},
}
tc.WriteTopology(node0.Path, top)
@ -380,8 +378,8 @@ func TestCluster_ResizeStates(t *testing.T) {
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != pilosa.ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateStarting, node0.State())
if node0.State() != ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
@ -395,10 +393,10 @@ func TestCluster_ResizeStates(t *testing.T) {
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State())
} else if node2.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node2.State())
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node2.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State())
}
// Close TestCluster.
@ -408,7 +406,7 @@ func TestCluster_ResizeStates(t *testing.T) {
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
@ -418,21 +416,20 @@ func TestCluster_ResizeStates(t *testing.T) {
}
// Add Bit Data to node0.
if err := tc.CreateFrame("i", "f", pilosa.FrameOptions{}); err != nil {
if err := tc.CreateFrame("i", "f", FrameOptions{}); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", "standard", 1, 101, nil)
tc.SetBit("i", "f", "standard", 1, 1300000, nil)
// Add Field Data to node0.
if err := tc.CreateFrame("i", "fields", pilosa.FrameOptions{
if err := tc.CreateFrame("i", "fields", FrameOptions{
InverseEnabled: false,
RangeEnabled: true,
//CacheType: pilosa.CacheTypeNone,
Fields: []*pilosa.Field{
//CacheType: CacheTypeNone,
Fields: []*Field{
{
Name: "fld0",
Type: pilosa.FieldTypeInt,
Type: FieldTypeInt,
Min: -100,
Max: 100,
},
@ -462,13 +459,13 @@ func TestCluster_ResizeStates(t *testing.T) {
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State())
} else if node1.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State())
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &pilosa.Topology{
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
@ -511,7 +508,7 @@ func TestCluster_ResizeStates(t *testing.T) {
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := test.NewCluster(2)
c := NewTestCluster(2)
oldNode := c.Nodes[0]
newNode := c.Nodes[1]

3
cmd/import.go
View file

@ -60,10 +60,9 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index (DEPRECATED. This feature will be removed in a future version. Set time quantum of each frame instead.)")
flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame")
flags.BoolVar(&Importer.FrameOptions.InverseEnabled, "frame-inverse-enabled", false, "Enable inverse frame")
flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "Enabled range encoded frame")
flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.")
flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked")
flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame")
ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify)

56
cmd/server.go
View file

@ -15,17 +15,11 @@
package cmd
import (
"fmt"
"io"
"os"
"os/signal"
"runtime/pprof"
"syscall"
"time"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/ctl"
"github.com/pilosa/pilosa/server"
)
@ -45,52 +39,10 @@ It will load existing data from the configured
directory and start listening for client connections
on the configured port.`,
RunE: func(cmd *cobra.Command, args []string) error {
// Set up the logger.
if err := Server.SetupLogger(); err != nil {
return fmt.Errorf("error setting up the logger: %v", err)
if err := Server.Start(); err != nil {
return errors.Wrap(err, "running server")
}
logger := Server.Server.Logger
logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime)
// Start CPU profiling.
if Server.CPUProfile != "" {
f, err := os.Create(Server.CPUProfile)
if err != nil {
return fmt.Errorf("create cpu profile: %v", err)
}
defer f.Close()
fmt.Fprintln(Server.Stderr, "Starting cpu profile")
pprof.StartCPUProfile(f)
time.AfterFunc(Server.CPUTime, func() {
fmt.Fprintln(Server.Stderr, "Stopping cpu profile")
pprof.StopCPUProfile()
f.Close()
})
}
// Execute the program.
if err := Server.Run(); err != nil {
return fmt.Errorf("error running server: %v", err)
}
// First SIGKILL causes server to shut down gracefully.
c := make(chan os.Signal, 2)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
select {
case sig := <-c:
logger.Printf("Received %s; gracefully shutting down...\n", sig.String())
// Second signal causes a hard shutdown.
go func() { <-c; os.Exit(1) }()
if err := Server.Close(); err != nil {
return err
}
case <-Server.Done:
logger.Printf("Server closed externally")
}
return nil
return errors.Wrap(Server.Wait(), "waiting on Server")
},
}

20
cmd/server_test.go
View file

@ -21,9 +21,9 @@ import (
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/cmd"
_ "github.com/pilosa/pilosa/test"
"github.com/pilosa/pilosa/toml"
)
func TestServerHelp(t *testing.T) {
@ -37,8 +37,6 @@ func TestServerHelp(t *testing.T) {
func TestServerConfig(t *testing.T) {
actualDataDir, err := ioutil.TempDir("", "")
failErr(t, err, "making data dir")
profFile, err := ioutil.TempFile("", "")
failErr(t, err, "making temp file")
logFile, err := ioutil.TempFile("", "")
failErr(t, err, "making log file")
tests := []commandTest{
@ -65,7 +63,7 @@ func TestServerConfig(t *testing.T) {
v.Check(cmd.Server.Config.Bind, "localhost:10111")
v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"})
v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90))
v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
return v.Error()
},
@ -86,14 +84,14 @@ func TestServerConfig(t *testing.T) {
validation: func() error {
v := validator{}
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"})
v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9))
v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9))
return v.Error()
},
},
// TEST 2
{
args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"},
env: map[string]string{"PILOSA_PROFILE_CPU_TIME": "1m"},
env: map[string]string{},
cfgFileContent: `
bind = "localhost:19444"
data-dir = "` + actualDataDir + `"
@ -103,9 +101,6 @@ func TestServerConfig(t *testing.T) {
]
[anti-entropy]
interval = "11m0s"
[profile]
cpu = "` + profFile.Name() + `"
cpu-time = "35s"
[metric]
service = "statsd"
host = "127.0.0.1:8125"
@ -113,9 +108,7 @@ func TestServerConfig(t *testing.T) {
validation: func() error {
v := validator{}
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"})
v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11))
v.Check(cmd.Server.CPUProfile, profFile.Name())
v.Check(cmd.Server.CPUTime, time.Minute)
v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*11))
v.Check(cmd.Server.Config.LogPath, logFile.Name())
v.Check(cmd.Server.Config.Metric.Service, "statsd")
v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125")
@ -147,6 +140,9 @@ func TestServerConfig(t *testing.T) {
case <-cmd.Server.Started:
case <-executed:
}
if execErr != nil {
t.Fatalf("executing server command: %v", execErr)
}
err := cmd.Server.Close()
failErr(t, err, "closing pilosa server command")
<-executed

251
config.go
View file

@ -1,251 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"time"
)
// Cluster types.
const (
ClusterNone = ""
ClusterStatic = "static"
ClusterGossip = "gossip"
)
const (
// DefaultDataDir is the default data directory.
DefaultDataDir = "~/.pilosa"
// DefaultHost is the default hostname to use.
DefaultHost = "localhost"
// DefaultPort is the default port to use with the hostname.
DefaultPort = "10101"
// DefaultClusterDisabled sets the node intercommunication method.
DefaultClusterDisabled = false
// DefaultMetrics sets the internal metrics to no-op.
DefaultMetrics = "nop"
// DefaultMaxWritesPerRequest is the default number of writes per request.
DefaultMaxWritesPerRequest = 5000
// Gossip config based on memberlist.Config.
// Port indicates the port to which pilosa should bind for internal state sharing.
DefaultGossipPort = "14000"
// StreamTimeout is the timeout for establishing a stream connection with
// a remote node for a full state sync, and for stream read and write
// operations. Maps to memberlist TCPTimeout.
DefaultGossipStreamTimeout = 10 * time.Second
// SuspicionMult is the multiplier for determining the time an
// inaccessible node is considered suspect before declaring it dead.
// The actual timeout is calculated using the formula:
//
// SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
//
// This allows the timeout to scale properly with expected propagation
// delay with a larger cluster size. The higher the multiplier, the longer
// an inaccessible node is considered part of the cluster before declaring
// it dead, giving that suspect node more time to refute if it is indeed
// still alive.
DefaultGossipSuspicionMult = 4
// PushPullInterval is the interval between complete state syncs.
// Complete state syncs are done with a single node over TCP and are
// quite expensive relative to standard gossiped messages. Setting this
// to zero will disable state push/pull syncs completely.
//
// Setting this interval lower (more frequent) will increase convergence
// speeds across larger clusters at the expense of increased bandwidth
// usage.
DefaultGossipPushPullInterval = 30 * time.Second
// ProbeInterval and ProbeTimeout are used to configure probing behavior
// for memberlist.
//
// ProbeInterval is the interval between random node probes. Setting
// this lower (more frequent) will cause the memberlist cluster to detect
// failed nodes more quickly at the expense of increased bandwidth usage.
//
// ProbeTimeout is the timeout to wait for an ack from a probed node
// before assuming it is unhealthy. This should be set to 99-percentile
// of RTT (round-trip time) on your network.
DefaultGossipProbeInterval = 1 * time.Second
DefaultGossipProbeTimeout = 500 * time.Millisecond
// Interval and Nodes are used to configure the gossip
// behavior of memberlist.
//
// Interval is the interval between sending messages that need
// to be gossiped that haven't been able to piggyback on probing messages.
// If this is set to zero, non-piggyback gossip is disabled. By lowering
// this value (more frequent) gossip messages are propagated across
// the cluster more quickly at the expense of increased bandwidth.
//
// Nodes is the number of random nodes to send gossip messages to
// per Interval. Increasing this number causes the gossip messages
// to propagate across the cluster more quickly at the expense of
// increased bandwidth.
//
// ToTheDeadTime is the interval after which a node has died that
// we will still try to gossip to it. This gives it a chance to refute.
DefaultGossipInterval = 200 * time.Millisecond
DefaultGossipNodes = 3
DefaultGossipToTheDeadTime = 30 * time.Second
DefaultMetricPollInterval = 0 * time.Minute
)
// TLSConfig contains TLS configuration
type TLSConfig struct {
// CertificatePath contains the path to the certificate (.crt or .pem file)
CertificatePath string `toml:"certificate-path"`
// CertificateKeyPath contains the path to the certificate key (.key file)
CertificateKeyPath string `toml:"certificate-key-path"`
// SkipVerify disables verification for self-signed certificates
SkipVerify bool `toml:"skip-verify"`
}
// Config represents the configuration for the command.
type Config struct {
DataDir string `toml:"data-dir"`
Bind string `toml:"bind"`
// Limits the number of mutating commands that can be in a single request to
// the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs.
MaxWritesPerRequest int `toml:"max-writes-per-request"`
LogPath string `toml:"log-path"`
Verbose bool `toml:"verbose"`
// TLS
TLS TLSConfig
Cluster struct {
Disabled bool `toml:"disabled"`
Coordinator bool `toml:"coordinator"`
ReplicaN int `toml:"replicas"`
Hosts []string `toml:"hosts"`
LongQueryTime Duration `toml:"long-query-time"`
} `toml:"cluster"`
Gossip struct {
Port string `toml:"port"`
Seeds []string `toml:"seeds"`
Key string `toml:"key"`
StreamTimeout Duration `toml:"stream-timeout"`
SuspicionMult int `toml:"suspicion-mult"`
PushPullInterval Duration `toml:"push-pull-interval"`
ProbeTimeout Duration `toml:"probe-timeout"`
ProbeInterval Duration `toml:"probe-interval"`
Nodes int `toml:"nodes"`
Interval Duration `toml:"interval"`
ToTheDeadTime Duration `toml:"to-the-dead-time"`
} `toml:"gossip"`
AntiEntropy struct {
Interval Duration `toml:"interval"`
} `toml:"anti-entropy"`
Metric struct {
Service string `toml:"service"`
Host string `toml:"host"`
PollInterval Duration `toml:"poll-interval"`
Diagnostics bool `toml:"diagnostics"`
} `toml:"metric"`
}
// NewConfig returns an instance of Config with default options.
func NewConfig() *Config {
c := &Config{
DataDir: DefaultDataDir,
Bind: ":" + DefaultPort,
MaxWritesPerRequest: DefaultMaxWritesPerRequest,
// LogPath: "",
// Verbose: false,
TLS: TLSConfig{},
}
// Cluster config.
c.Cluster.Disabled = DefaultClusterDisabled
// c.Cluster.Coordinator = false
c.Cluster.ReplicaN = DefaultReplicaN
c.Cluster.Hosts = []string{}
c.Cluster.LongQueryTime = Duration(time.Minute)
// Gossip config.
// c.Gossip.Port = ""
// c.Gossip.Seeds = []string{}
// c.Gossip.Key = ""
c.Gossip.StreamTimeout = Duration(DefaultGossipStreamTimeout)
c.Gossip.SuspicionMult = DefaultGossipSuspicionMult
c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval)
c.Gossip.ProbeTimeout = Duration(DefaultGossipProbeTimeout)
c.Gossip.ProbeInterval = Duration(DefaultGossipProbeInterval)
c.Gossip.Nodes = DefaultGossipNodes
c.Gossip.Interval = Duration(DefaultGossipInterval)
c.Gossip.ToTheDeadTime = Duration(DefaultGossipToTheDeadTime)
// AntiEntropy config.
c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval)
// Metric config.
c.Metric.Service = DefaultMetrics
// c.Metric.Host = ""
c.Metric.PollInterval = Duration(DefaultMetricPollInterval)
c.Metric.Diagnostics = true
return c
}
// Validate that all configuration permutations are compatible with each other.
func (c *Config) Validate() error {
if !c.Cluster.Disabled && len(c.Cluster.Hosts) > 0 {
return ErrConfigClusterEnabledHosts
}
return nil
}
// Duration is a TOML wrapper type for time.Duration.
type Duration time.Duration
// String returns the string representation of the duration.
func (d Duration) String() string { return time.Duration(d).String() }
// UnmarshalText parses a TOML value into a duration value.
func (d *Duration) UnmarshalText(text []byte) error {
v, err := time.ParseDuration(string(text))
if err != nil {
return err
}
*d = Duration(v)
return nil
}
// MarshalText writes duration value in text format.
func (d Duration) MarshalText() (text []byte, err error) {
return []byte(d.String()), nil
}
// MarshalTOML write duration into valid TOML.
func (d Duration) MarshalTOML() ([]byte, error) {
return []byte(d.String()), nil
}

5
ctl/backup.go
View file

@ -21,6 +21,7 @@ import (
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
)
// BackupCommand represents a command for backing up a view.
@ -39,7 +40,7 @@ type BackupCommand struct {
// Standard input/output
*pilosa.CmdIO
TLS pilosa.TLSConfig
TLS server.TLSConfig
}
// NewBackupCommand returns a new instance of BackupCommand.
@ -88,6 +89,6 @@ func (cmd *BackupCommand) TLSHost() string {
return cmd.Host
}
func (cmd *BackupCommand) TLSConfiguration() pilosa.TLSConfig {
func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

10
ctl/backup_test.go
View file

@ -50,12 +50,10 @@ func TestBackupCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].URI = *uri
s.Handler.Holder = hldr.Holder
s.Handler.API.URI = *uri
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm := NewBackupCommand(stdin, stdout, stderr)
file, err := ioutil.TempFile("", "import.csv")

5
ctl/bench.go
View file

@ -24,6 +24,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/server"
)
// BenchCommand represents a command for benchmarking index operations.
@ -42,7 +43,7 @@ type BenchCommand struct {
// Standard input/output
*pilosa.CmdIO
TLS pilosa.TLSConfig
TLS server.TLSConfig
}
// NewBenchCommand returns a new instance of BenchCommand.
@ -110,6 +111,6 @@ func (cmd *BenchCommand) TLSHost() string {
return cmd.Host
}
func (cmd *BenchCommand) TLSConfiguration() pilosa.TLSConfig {
func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

5
ctl/common.go
View file

@ -18,13 +18,14 @@ import (
"crypto/tls"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/spf13/pflag"
)
// CommandWithTLSSupport is the interface for commands which has TLS settings
type CommandWithTLSSupport interface {
TLSHost() string
TLSConfiguration() pilosa.TLSConfig
TLSConfiguration() server.TLSConfig
}
// SetTLSConfig creates common TLS flags
@ -48,7 +49,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error
InsecureSkipVerify: tlsConfig.SkipVerify,
}
}
client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), pilosa.GetHTTPClient(TLSConfig))
client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig))
if err != nil {
return nil, err
}

3
ctl/config.go
View file

@ -21,12 +21,13 @@ import (
toml "github.com/pelletier/go-toml"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
)
// ConfigCommand represents a command for printing a default config.
type ConfigCommand struct {
*pilosa.CmdIO
Config *pilosa.Config
Config *server.Config
}
// NewConfigCommand returns a new instance of ConfigCommand.

4
ctl/config_test.go
View file

@ -22,7 +22,7 @@ import (
"strings"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
)
func TestConfigCommand_Run(t *testing.T) {
@ -30,7 +30,7 @@ func TestConfigCommand_Run(t *testing.T) {
stdin := bytes.NewReader(rder)
r, w, _ := os.Pipe()
cm := NewConfigCommand(stdin, w, os.Stderr)
cm.Config = pilosa.NewConfig()
cm.Config = server.NewConfig()
err := cm.Run(context.Background())
w.Close()

5
ctl/export.go
View file

@ -21,6 +21,7 @@ import (
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
)
// ExportCommand represents a command for bulk exporting data from a server.
@ -38,7 +39,7 @@ type ExportCommand struct {
// Standard input/output
*pilosa.CmdIO
TLS pilosa.TLSConfig
TLS server.TLSConfig
}
// NewExportCommand returns a new instance of ExportCommand.
@ -114,6 +115,6 @@ func (cmd *ExportCommand) TLSHost() string {
return cmd.Host
}
func (cmd *ExportCommand) TLSConfiguration() pilosa.TLSConfig {
func (cmd *ExportCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

10
ctl/export_test.go
View file

@ -63,12 +63,10 @@ func TestExportCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0] = node
s.Handler.Holder = hldr.Holder
s.Handler.API.URI = *uri
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))

3
ctl/generate_config_test.go
View file

@ -17,7 +17,6 @@ package ctl
import (
"bytes"
"context"
"github.com/pilosa/pilosa"
"io"
"os"
"strings"
@ -35,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) {
io.Copy(&buf, r)
if err != nil {
t.Fatalf("Config Run doesn't work: %s", err)
} else if !strings.Contains(buf.String(), pilosa.DefaultHost) {
} else if !strings.Contains(buf.String(), "localhost:10101") {
t.Fatalf("Unexpected config: %s", buf.String())
}
}

5
ctl/import.go
View file

@ -27,6 +27,7 @@ import (
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
)
// ImportCommand represents a command for bulk importing data.
@ -66,7 +67,7 @@ type ImportCommand struct {
// Standard input/output
*pilosa.CmdIO
TLS pilosa.TLSConfig
TLS server.TLSConfig
}
// NewImportCommand returns a new instance of ImportCommand.
@ -447,6 +448,6 @@ func (cmd *ImportCommand) TLSHost() string {
return cmd.Host
}
func (cmd *ImportCommand) TLSConfiguration() pilosa.TLSConfig {
func (cmd *ImportCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

27
ctl/import_test.go
View file

@ -69,12 +69,10 @@ func TestImportCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0] = node
s.Handler.Holder = hldr.Holder
s.Handler.API.URI = *uri
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
cm.Index = "i"
@ -88,9 +86,7 @@ func TestImportCommand_Run(t *testing.T) {
}
// Ensure that the ImportValue path runs (note: we have specified a value
// for cm.Field. Because the handler doesn't return errors (it sends them
// to the logger), we don't get an error returned at `cm.Run()` even though
// we haven't setup frame `f` to be RangeEnabled.
// for cm.Field.)
func TestImportCommand_RunValue(t *testing.T) {
buf := bytes.Buffer{}
@ -111,16 +107,15 @@ func TestImportCommand_RunValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0] = node
s.Handler.Holder = hldr.Holder
s.Handler.API.URI = *uri
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`)))
cm.Index = "i"
cm.Frame = "f"
@ -137,7 +132,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
cm.Host = pilosa.DefaultHost
cm.Host = "anyhost"
cm.Index = "i"
cm.Frame = "f"
file, err := ioutil.TempFile("", "import.csv")

5
ctl/restore.go
View file

@ -21,6 +21,7 @@ import (
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
)
// RestoreCommand represents a command for restoring a frame from a backup.
@ -39,7 +40,7 @@ type RestoreCommand struct {
// Standard input/output
*pilosa.CmdIO
TLS pilosa.TLSConfig
TLS server.TLSConfig
}
// NewRestoreCommand returns a new instance of RestoreCommand.
@ -81,6 +82,6 @@ func (cmd *RestoreCommand) TLSHost() string {
return cmd.Host
}
func (cmd *RestoreCommand) TLSConfiguration() pilosa.TLSConfig {
func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

10
ctl/restore_test.go
View file

@ -52,12 +52,10 @@ func TestRestoreCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].URI = *uri
s.Handler.Holder = hldr.Holder
s.Handler.API.URI = *uri
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm := NewRestoreCommand(stdin, stdout, stderr)
cm.Path = file.Name()

4
ctl/server.go
View file

@ -61,8 +61,4 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.")
flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
// CPU Profiling
flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.")
flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.")
}

22
diagnostics.go
View file

@ -168,23 +168,23 @@ func (d *DiagnosticsCollector) logErr(err error) bool {
// EnrichWithOSInfo adds OS information to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithOSInfo() {
uptime, err := d.server.SystemInfo.Uptime()
uptime, err := d.server.systemInfo.Uptime()
if !d.logErr(err) {
d.Set("HostUptime", uptime)
}
platform, err := d.server.SystemInfo.Platform()
platform, err := d.server.systemInfo.Platform()
if !d.logErr(err) {
d.Set("OSPlatform", platform)
}
family, err := d.server.SystemInfo.Family()
family, err := d.server.systemInfo.Family()
if !d.logErr(err) {
d.Set("OSFamily", family)
}
version, err := d.server.SystemInfo.OSVersion()
version, err := d.server.systemInfo.OSVersion()
if !d.logErr(err) {
d.Set("OSVersion", version)
}
kernelVersion, err := d.server.SystemInfo.KernelVersion()
kernelVersion, err := d.server.systemInfo.KernelVersion()
if !d.logErr(err) {
d.Set("OSKernelVersion", kernelVersion)
}
@ -192,15 +192,15 @@ func (d *DiagnosticsCollector) EnrichWithOSInfo() {
// EnrichWithMemoryInfo adds memory information to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
memFree, err := d.server.SystemInfo.MemFree()
memFree, err := d.server.systemInfo.MemFree()
if !d.logErr(err) {
d.Set("MemFree", memFree)
}
memTotal, err := d.server.SystemInfo.MemTotal()
memTotal, err := d.server.systemInfo.MemTotal()
if !d.logErr(err) {
d.Set("MemTotal", memTotal)
}
memUsed, err := d.server.SystemInfo.MemUsed()
memUsed, err := d.server.systemInfo.MemUsed()
if !d.logErr(err) {
d.Set("MemUsed", memUsed)
}
@ -219,10 +219,8 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
numIndexes += 1
for _, frame := range index.Frames() {
numFrames += 1
if frame.rangeEnabled {
if fields, err := frame.GetFields(); err == nil {
bsiFieldCount += len(fields)
}
if fields, err := frame.GetFields(); err == nil {
bsiFieldCount += len(fields)
}
if frame.TimeQuantum() != "" {
timeQuantumEnabled = true

43
docs/administration.md
View file

@ -48,9 +48,9 @@ On Mac OS X, `ulimit` does not behave predictably. [This blog post](https://blog
#### Importing
The import API expects a csv of RowID,ColumnID's.
The import API expects a csv of rowID,columnID's.
When importing large datasets remember it is much faster to pre sort the data by RowID and then by ColumnID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results.
When importing large datasets remember it is much faster to pre sort the data by row ID and then by column ID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results.
```
pilosa import --sort -i project -f stargazer project-stargazer.csv
@ -70,7 +70,7 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count
#### Exporting
Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format RowID,ColumnID and sorted by column ID.
Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format rowID,columnID and sorted by columnID.
```
curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0&view=standard" \
--header "Accept: text/csv"
@ -92,7 +92,41 @@ The Pilosa server should support PQL versioning using HTTP headers. On each requ
#### Upgrading
When upgrading, upgrade clients first, followed by server for all Minor and Patch level changes.
To upgrade Pilosa:
1. First, upgrade the [client libraries](../client-libraries/) you are using in your application. Generally, a client version `X` will be compatible with the Pilosa server version `X` and earlier. For example, `python-pilosa 0.9.0` is compatible with both `pilosa 0.8.0` and `pilosa 0.9.0`.
2. Next, download the latest release from our [installation page](/docs/latest/installation/) or from the [release page on Github](https://github.com/pilosa/pilosa/releases).
3. Shut down the Pilosa cluster.
4. Make a backup of the [data directory](../configuration/#data-dir) on each cluster node.
5. Upgrade the Pilosa server binaries and any configuration changes. See the following sections on any version-specific changes you must make.
6. Start Pilosa. It is recommended to start the cluster coordinator node first, followed by any other nodes.
##### Version 0.9
Pilosa v0.9 introduces a few compatibility changes that need to be addressed.
**Configuration changes**: These changes need to occur before starting Pilosa v0.9:
1. Cluster-resize capability eliminates the `hosts` setting. Now, cluster membership is determined by `gossip`. This is only a factor if you are running Pilosa as a cluster.
2. Gossip-based cluster membership requires you to set a single cluster node as a [coordinator](../configuration/#cluster-coordinator). Make sure only a single node has the `cluster.coordinator` flag set.
3. `gossip.seed` has been renamed [`gossip.seeds`](../configuration/#gossip-seeds) and takes multiple items. It is recommended that at least two nodes are specified as gossip seeds.
**Data directory changes**: These changes need to occur while the cluster is shut down, before starting Pilosa v0.9:
Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topology` file. Due to the way Pilosa internally shards indices, upgrading a Pilosa cluster will result in data loss if an existing cluster is brought up without these files. New clusters will generate them automatically, but you may migrate an existing cluster by using a tool we called `topology-generator`:
1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard (AKA slice) ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical.
2. Install the `topology-generator`: `go get github.com/pilosa/upgrade-utils/v0.9/topology-generator`.
3. Run the `topology-generator`. There are two arguments: the number of nodes and the output directory. For this example, we'll assume a 3-node cluster and place the files in the current working directory: `topology-generator 3 .`.
4. This tool will generate a file, `topology`, and multiple id files, called `nodeX.id`, X being the node index position.
5. Copy the file `topology` into the data directories of every node in the cluster, naming it `.topology` (note the dot), e.g. `cp topology ~/.pilosa/.topology` or `scp topology node1:.pilosa/.topology`.
6. Copy the node ID files into the respective node data directories. For example, `node0.id` will be placed on the first node in the `cluster.hosts` list, with the name `.id`. For example: `scp node0.id node0:.pilosa/.id`. Again, it is very important that the ordering you give the nodes with these IDs matches the ordering you had in your existing `cluster.hosts` setting.
**Application changes**:
1. Row and column labels were deprecated in Pilosa v0.8, and removed in Pilosa v0.9. Make sure that your application does not attempt to use a custom row or column label, as they are no longer supported.
2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-frame, as index-level time-quantums have been removed.
3. Inverse frames have been deprecated, removed from docs, and will be unsupported in the next release.
### Resizing the Cluster
@ -201,7 +235,6 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi
- **NumCPU:** Number of Cores per Node
- **BSIEnabled:** Bit Slice Index Frames in use.
- **TimeQuantumEnabled:** Time Quantum Frames in use.
- **InverseEnabled:** Inverse Frames in use.
- **NumIndexes:** Number of Indexes in the Cluster.
- **NumFrames:** Number of Frames in the Cluster.
- **NumSlices:** Number of Slices in the Cluster.

165
docs/api-reference.md
View file

@ -68,7 +68,7 @@ Sends a query to the Pilosa server with the given index. The request body is UTF
``` request
curl localhost:10101/index/user/query \
-X POST \
-d 'Bitmap(frame="language", rowID=5)'
-d 'Bitmap(frame="language", row=5)'
```
``` response
{"results":[{"attrs":{},"bits":[100]}]}
@ -83,7 +83,7 @@ The query is executed for all [slices](../data-model/#slice) by default. To use
``` request
curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \
-X POST \
-d 'Bitmap(frame="language", rowID=5)'
-d 'Bitmap(frame="language", row=5)'
```
``` response
{
@ -103,10 +103,9 @@ Creates a frame in the given index with the given name.
The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields:
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`.
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
* `rangeEnabled` (boolean): Enables range-encoded fields in this frame.
* `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields.
* `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding).
Each individual `field` contains the following:
@ -119,9 +118,7 @@ Each individual `field` contains the following:
Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
``` request
curl localhost:10101/index/user/frame/language \
-X POST \
-d '{"options": {"inverseEnabled": true}}'
curl localhost:10101/index/user/frame/language -X POST
```
``` response
{}
@ -130,7 +127,7 @@ curl localhost:10101/index/user/frame/language \
``` request
curl localhost:10101/index/repository/frame/stats \
-X POST \
-d '{"rangeEnabled": true, "fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
-d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
```
``` response
{}
@ -149,35 +146,6 @@ curl -XDELETE localhost:10101/index/user/frame/language
{}
```
### Change frame time quantum
`PATCH /index/<index-name>/frame/<frame-name>/time-quantum`
Changes the time quantum for the given frame. This endpoint should be called at most once right after creating a frame.
The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Valid time quantum values are:
* (Empty string)
* Y: year
* M: month
* D: day
* H: hour
* YM: year and month
* MD: month and day
* DH: day and hour
* YMD: year, month and day
* MDH: month, day and hour
* YMDH: year, month, day and hour
``` request
curl localhost:10101/index/user/frame/language/time-quantum \
-X POST \
-d '{"timeQuantum": "YM"}'
```
``` response
{}
```
### Create Field
`POST /index/<index-name>/frame/<frame-name>/field/<field-name>`
@ -198,129 +166,6 @@ curl localhost:10101/index/repository/frame/stats/field/pullrequests \
{}
```
### Create input definition
<div class="warning">
Input definition is deprecated as of v0.9.
</div>
`POST /index/<index-name>/input-definition/<input-definition-name>`
Creates an input definition in the given index with the given name.
The request payload is JSON, and it must contain the fields `frames` and `fields`. `frames` is an array of frames used within this input definition. Each frame must contain a `name` and may contain the following options:
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`.
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field's value must be an unsigned integer which maps directly to a columnID in Pilosa.
* `name` (string): Maps the source data field to actions that process the field's corresponding value.
* `actions` (array): List of actions that will process the field's value.
The `action` describes how the field value will be processed. Each `action` may contain:
* `frame` (string): The Frame that will contain this action's set bits.
* `rowid` (int): The action can use this as a pre-defined SetBit rowID. The user is required to ensure this ID does not overlap with other rows in use per frame.
* `valueDestination` (string): The mapping rule used for this data.
- `value-to-row`: The value should be an integer and will map directly to a RowID.
- `single-row-boolean`: If the value is true set a bit using the `rowid`.
- `mapping`: Map the value to a RowID in the `valueMap`.
* `valueMap` (object): string and integer pairs used to map field values to RowID's.
``` request
curl localhost:10101/index/user/input-definition/stargazer-input \
-X POST \
-d '{
"frames":[
{
"name": "language",
"options": {"inverseEnabled": true}
}
],
"fields":[
{
"name": "repo_id",
"primaryKey":true
},
{
"name": "language_id",
"actions":[
{
"frame": "language",
"valueDestination": "mapping",
"valueMap": {
"Go": 5,
"Python": 17,
"C++": 10
}
}
]
}
]
}'
```
``` response
{}
```
### Get input definition
<div class="warning">
Input definition is deprecated as of v0.9.
</div>
`GET /index/<index-name>/input-definition/<input-definition-name>`
Returns the given input definition as JSON.
``` request
curl -XGET localhost:10101/index/user/input-definition/stargazer-input
```
``` response
{"frames":[{"name":"language","options":{"inverseEnabled":true}}],"fields":[{"name":"repo_id","primaryKey":true},{"name":"language_id","actions":[{"frame":"language","valueDestination":"mapping","valueMap":{"Go":5,"Python":17,"C++":10}}]}]}
```
### Remove input definition
<div class="warning">
Input definition is deprecated as of v0.9.
</div>
`DELETE /index/<index-name>/input-definition/<input-definition-name>`
Removes the given input definition.
``` request
curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input
```
``` response
{}
```
### Process input data
<div class="warning">
Input definition is deprecated as of v0.9.
</div>
`POST /index/<index-name>/input/<input-definition-name>`
Processes the JSON payload using the given input definition.
The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column, and additional fields that will be handled by corresponding actions in the input definition.
``` request
curl localhost:10101/index/user/input/stargazer-input \
-X POST \
-d '[{"language_id": "Go", "repo_id": 92274475}]'
```
``` response
{}
```
### List hosts
`GET /hosts`

10
docs/configuration.md
View file

@ -197,12 +197,12 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
```toml
[profile]
cpu = "/path/to/somewhere"
cpu = "/path/to/somewhere"
```
#### Profile CPU Time
* Description: Amount of time to collect cpu profiling data if `profile.cpu` is set.
* Description: Amount of time to collect cpu profiling data at startup if `profile.cpu` is set.
* Flag: `--profile.cpu-time="30s"`
* Env: `PILOSA_PROFILE_CPU_TIME="30s"
* Config:
@ -213,7 +213,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
```
#### Metric Service
* Description: Which stats service to use. Choose from [statsd, expvar].
* Description: Which stats service to use. Choose from [statsd, expvar, none].
* Flag: `--metric.service=statsd`
* Env: `PILOSA_METRIC_SERVICE=statsd'
* Config:
@ -236,7 +236,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
#### Metric Poll Interval
* Description: Polling interval for runtime metrics.
* Description: Rate at which runtime metrics (such as open file handles and memory usage) are collected.
* Flag: `metric.poll-interval=”0m15s”`
* Env: `PILOSA_METRIC_POLL_INTERVAL=0m15s`
* Config:
@ -248,7 +248,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
#### Metric Diagnostics
* Description: Enable diagnostic reporting. To disable diagnostics set to false.
* Description: Enable reporting of limited usage statistics to Pilosa developers. To disable, set to false.
* Flag: `metric.diagnostics`
* Env: `PILOSA_METRIC_DIAGNOSTICS`
* Config:

36
docs/data-model.md
View file

@ -77,33 +77,19 @@ Columns are sharded on a preset width, and each shard is referred to as a Slice.
### View
Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. The Inverse View contains the same data with the axes inverted.Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation.
Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation.
#### Standard
The standard View contains the same Row/Column format as the input data.
#### Inverse
The Inverse View contains the same data with the Row and Column swapped.
For example, the following `SetBit()` queries will result in the data described in the illustration below:
```
SetBit(frame="A", rowID=8, columnID=3)
SetBit(frame="A", rowID=11, columnID=3)
SetBit(frame="A", rowID=19, columnID=5)
```
![inverse frame diagram](/img/docs/frame-inverse.svg)
*Inverse frame diagram*
#### Time Quantums
If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the illustration below:
```
SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-18T00:00")
SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00")
SetBit(frame="A", row=8, col=3, timestamp="2017-05-18T00:00")
SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00")
```
![time quantum frame diagram](/img/docs/frame-time-quantum.svg)
@ -112,19 +98,19 @@ SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00")
#### BSI Range-Encoding
Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded
bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers.
bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers.
Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rowIDs' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
For example, the following `SetFieldValue()` queries will result in the data described in the illustration below:
```
SetFieldValue(columnID=1, frame="A", field0=1)
SetFieldValue(columnID=2, frame="A", field0=2)
SetFieldValue(columnID=3, frame="A", field0=3)
SetFieldValue(columnID=4, frame="A", field0=7)
SetFieldValue(columnID=2, frame="A", field1=1)
SetFieldValue(columnID=3, frame="A", field1=6)
SetFieldValue(col=1, frame="A", field0=1)
SetFieldValue(col=2, frame="A", field0=2)
SetFieldValue(col=3, frame="A", field0=3)
SetFieldValue(col=4, frame="A", field0=7)
SetFieldValue(col=2, frame="A", field1=1)
SetFieldValue(col=3, frame="A", field1=6)
```
![BSI frame diagram](/img/docs/frame-bsi.svg)

8
docs/examples.md
View file

@ -201,11 +201,15 @@ For more examples and details, see this [ipython notebook](https://github.com/pi
### Chemical similarity search
<div class="warning">
This example uses the inverse frames feature, which is deprecated as of v0.9.0. This will soon be updated to reflect the current Pilosa API.
</div>
#### Overview
The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals.
We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set.
We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. Others have tried to solve this chemical similarity search problem using databases (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set.
Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient.
```
@ -218,7 +222,7 @@ All source code to calculate tanimoto for molecule fingerprint using Pilosa is a
#### Data model
We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id.
We use the [latest ChEMBL release](ftp://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/) chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id.
Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions.

27
docs/getting-started.md
View file

@ -4,7 +4,6 @@ weight = 3
nav = [
"Starting Pilosa",
"Sample Project",
"Input Definition",
"What's Next?",
]
+++
@ -111,14 +110,6 @@ docker exec -it pilosa /pilosa import -i repository -f language /language.csv
Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
### Input Definition
<div class="warning">
Input definition is deprecated as of v0.9.
</div>
Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data.
#### Make Some Queries
<div class="note">
@ -129,7 +120,7 @@ Which repositories did user 14 star:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Bitmap(frame="stargazer", rowID=14)'
-d 'Bitmap(frame="stargazer", row=14)'
```
``` response
{
@ -167,8 +158,8 @@ Which repositories were starred by user 14 and 19:
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(
Bitmap(frame="stargazer", rowID=14),
Bitmap(frame="stargazer", rowID=19)
Bitmap(frame="stargazer", row=14),
Bitmap(frame="stargazer", row=19)
)'
```
``` response
@ -187,8 +178,8 @@ Which repositories were starred by user 14 or 19:
curl localhost:10101/index/repository/query \
-X POST \
-d 'Union(
Bitmap(frame="stargazer", rowID=14),
Bitmap(frame="stargazer", rowID=19)
Bitmap(frame="stargazer", row=14),
Bitmap(frame="stargazer", row=19)
)'
```
``` response
@ -207,9 +198,9 @@ Which repositories were starred by user 14 and 19 and also were written in langu
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(
Bitmap(frame="stargazer", rowID=14),
Bitmap(frame="stargazer", rowID=19),
Bitmap(frame="language", rowID=1)
Bitmap(frame="stargazer", row=14),
Bitmap(frame="stargazer", row=19),
Bitmap(frame="language", row=1)
)'
```
``` response
@ -227,7 +218,7 @@ Set user 99999 as a stargazer for repository 77777:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", columnID=77777, rowID=99999)'
-d 'SetBit(frame="stargazer", column=77777, row=99999)'
```
``` response
{"results":[true]}

12
docs/glossary.md
View file

@ -14,7 +14,7 @@ nav = []
<strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap.
<strong id="bsi">[BSI](../data-model/#bsi-range-encoding)</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi) and [Sum](#sum) queries.
<strong id="bsi">[BSI](../data-model/#bsi-range-encoding)</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries.
<strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
@ -24,14 +24,18 @@ nav = []
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index).
<strong id="frame">[Frame](../data-model/#frame):</strong> Frames are used to group [rows](#row) into different categories. `RowID`s are namespaced by frame such that the same `RowID` in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame.
<strong id="frame">[Frame](../data-model/#frame):</strong> Frames are used to group [rows](#row) into different categories. Row IDs are namespaced by frame such that the same row ID in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame.
<strong id="index">[Index](../data-model/#index):</strong> An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Queries cannot operate across multiple indexes.
<strong id="jump-consistent-hash">[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf):</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
<strong id="max">[Max](../query-language/#max):</strong> A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field).
<strong id="maxslice">MaxSlice:</strong> The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries.
<strong id="min">[Min](../query-language/#min):</strong> A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field).
<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a [cluster](#cluster).
<strong id="partition">Partition:</strong> The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data.
@ -62,6 +66,6 @@ nav = []
<strong id="toml">[TOML](https://github.com/toml-lang/toml):</strong> the language used for Pilosa's [configuration file](../configuration/).
<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame).
<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame).
<strong id="view">[View](../data-model/#view):</strong> Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.
<strong id="view">[View](../data-model/#view):</strong> Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.

11
docs/input-definition.md
View file

@ -1,11 +0,0 @@
+++
title = "Input Definition"
weight = 8
+++
## Input Definition
<div class="warning">
Input definition is deprecated as of Pilosa v0.9.<br/>
<br/>
The previous version of this page is still available <a href="https://www.pilosa.com/docs/v0.8/input-definition">here</a>.
</div>

12
docs/installation.md
View file

@ -74,19 +74,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
1. Download the latest release:
```
curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-darwin-amd64.tar.gz
curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.9.0/pilosa-v0.9.0-darwin-amd64.tar.gz
```
Other releases can be downloaded from our Releases page on Github.
2. Extract the binary:
```
tar xfz pilosa-v0.8.3-darwin-amd64.tar.gz
tar xfz pilosa-v0.9.0-darwin-amd64.tar.gz
```
3. Move the binary into your PATH so you can run `pilosa` from any shell:
```
cp -i pilosa-v0.8.3-darwin-amd64/pilosa /usr/local/bin
cp -i pilosa-v0.9.0-darwin-amd64/pilosa /usr/local/bin
```
4. Make sure Pilosa is installed successfully:
@ -234,19 +234,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
1. To install the latest version of Pilosa, download the latest release:
```
curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-linux-amd64.tar.gz
curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.9.0/pilosa-v0.9.0-linux-amd64.tar.gz
```
Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github.
2. Extract the binary:
```
tar xfz pilosa-v0.8.3-linux-amd64.tar.gz
tar xfz pilosa-v0.9.0-linux-amd64.tar.gz
```
3. Move the binary into your PATH so you can run `pilosa` from any shell:
```
cp -i pilosa-v0.8.3-linux-amd64/pilosa /usr/local/bin
cp -i pilosa-v0.9.0-linux-amd64/pilosa /usr/local/bin
```
4. Make sure Pilosa is installed successfully:

110
docs/pdk.md
View file

@ -2,66 +2,72 @@
title = "PDK"
weight = 11
nav = [
"Examples and Executables",
"Library",
"Examples",
]
+++
## PDK
The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains Go libraries to help you use Pilosa effectively. From importing data quickly, to managing the mappings from contiguous integer ids to values of other types, the PDK should help you get off the ground quickly.
The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains executables, examples, and Go libraries to help you use Pilosa effectively.
### Examples and Executables
Running `pdk -h` will give the most up to date list of all the tools and examples that PDK provides. We'll cover a few of the more important ones here.
#### Kafka
`pdk kafka` reads either JSON or Avro encoded records from Kafka (using the
Confluent Schema Registry in the case of Avro), and indexes them in Pilosa. Each
record from Kafka is assigned a Pilosa column, and each value in a record is
assigned a row or field. Frame and field names are built from the "path" through
the record to arrive at that field. For example:
```json
{
"name": "jill",
"favorite_foods": ["corn chips", "chipotle dip"],
"location": {
"city": "Austin",
"state": "Texas",
"latitude": 3754,
"longitude": 4526
},
"active": true,
"age": 27
}
```
This JSON object would result in the following Pilosa schema:
| Name | Field | Type | Size/Min | Max |
|----------------|-----------|--------|----------|------------|
| name | | ranked | 100000 | |
| favorite_foods | | ranked | 100000 | |
| default | | Ranked | 100000 | |
| | age | int | 0 | 2147483647 |
| location | | ranked | 1000 | |
| | latitude | int | 0 | 2147483647 |
| | longitude | int | 0 | 2147483647 |
| location-city | | ranked | 100000 | |
| location-state | | ranked | 100000 | |
All frames are created as ranked frames by default, and fields are created with
a minmum size of zero and a fixed maximum of 2147483647. Fields at the top level
are created in the default frame. Frames are a dash-separated concatenation of
all key values in the path - you can see this with frames like location-city.
Most the options to `pdk kafka` are self-explanatory (kafka hosts, pilosa hosts,
kafka topics, kafka group, etc.), but there are a few options that give some
control over the way data is indexed, and ingestion performance.
* `--batch-size`: The batch size control how many set bits or values are batched up to be imported *per frame*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner.
* `--framer.collapse`: This is a list of strings which will be removed from the frame names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be frames named "city" and "state" rather than "location-city" and "location-state".
* `--framer.ignore`: This allows you to skip indexing on any path containing these strings. If you have a field like email address or some other unique ID, you might not want to index it.
* `--subject-path`: If nothing is passed for this option, then each record will be assigned a unique sequential column ID. If `subject-path` is specified, then the value at this path in the record will be mapped to a column ID. If the same value appears in another record, the same column ID will be used.
* `--proxy`: The PDK ingests data, but also keeps a mapping for string values to row IDs, and from subjects to column ids. Because of this, querying Pilosa directly may not be useful, since it only returns integer row and column ids. The PDK will start a proxy server which intercepts requests to Pilosa using strings for row and column ids, and translates them to the integers that Pilosa understands. It will also translate responses so that (e.g.) a TopN query will return `{"results":[[{"Key":"chipotle dip","Count":1},{"Key":"corn chips","Count":1}]]}`. By default, the mapping is stored in an embedded leveldb.
The PDK also contains some fully worked examples which make use of its tools. These are available in the `usecase` subdirectory and can be run as subcommands of the `pdk` binary.
### Library
#### Mapping
For now, the [Godocs](https://godoc.org/github.com/pilosa/pdk) have the most up to date library documentation.
Importing data into Pilosa is dependent on mapping it to integer IDs. PDK provides some predefined functions for inline mapping to simplify this process, supported by a framework for linking these mappings with the associated fields in a source CSV file. If no custom mapping is necessary, the entire import process can be described by an import definition file. The file is composed of four main parts:
* an enumeration of field names
* a list of parsers that are used to parse strings in the CSV to values
* a list of commonly used, named, mapper functions
* a list of ParserMappers - objects that encapsulate all of the work related to a single frame.
This definition file can quickly get long, and defining it manually would be quite tedious. That's why we have a tool to generate a definition file by looking at a data set. This will handle most of the legwork, but since it can only guess at the application, it uses the simplest mappings - each column gets mapped to one frame in an appropriate way. This is intended as a starting point, to be updated to suit your use of the PDK.
With this definition available, the PDK tool can run the import, which consists of these steps:
- create the index
- create all frames
- for each CSV file, read all rows
- for each CSV record:
- generate a columnID
- apply all ParserMappers, generating a list of (frame, ID) pairs
- set the appropriate bit. schematically: SetBit(id=rowID, frame=frame, profileID=columnID)
The process is summarized in this flowchart:
![Bitmapper flowchart](/img/docs/pdk-bitmapper-flowchart.svg)
Some of the simple mapper functions available with PDK include:
* YearMapper: Maps a `time.Time` value to an integer equal to the `Time`'s year.
* MonthMapper: Maps a `time.Time` value to an integer equal to the `Time`'s month, in [0, 11].
* DayOfWeekMapper: Maps a `time.Time` value to an integer equal to the `Time`'s day of the week, in [0, 6].
* HourMapper: Maps a `time.Time` value to an integer equal to the `Time`'s hour, in [0, 23].
* TimeOfDayMapper: Maps a `time.Time` value to the range [0, N-1], where N is specified by `Res`. This is useful if the resolution used by HourMapper is too small (or large). For example, TimeOfDayMapper with `Res`=48 maps to 48 half-hour bins.
* BoolMapper: Maps a boolean value to the range [0, 1].
* IntMapper: Maps an integer value to the range [Min, Max]. This is suitable for a field with a small- to moderate-sized domain.
* SparseIntMapper: Maps integer values through an arbitrary table, foreign keys for example. This is suitable if the table size is small.
* LinearFloatMapper: Maps floating point values through a linear function. Inputs in the range [`Min`, `Max`] are mapped to row IDs in the range [0, `Res - 1`], where each ID represents one of `Res` evenly spaced buckets.
* FloatMapper: Maps floating point values using arbitrary buckets, in case even spacing is not suitable. These buckets are specified with an array of floats representing the left end of each bucket.
* GridMapper: Maps a pair of floats to a single integer, identifying a cell in a rectangular grid. This can be used, for example, to represent (latitude, longitude) location coarsely, as in the taxi data example.
* CustomMapper: When none of the predefined mappers will work, or when multiple fields determine a row ID value, an arbitrary mapping function can be used. Define a function in Go, with the necessary behavior, and wrap it in a CustomMapper.
### Examples
Run `make install` to build and install the `pdk` binary which contains all the examples. Just running `pdk` will bring up a list of all the examples, with a brief description of each. `pdk help <example>` will bring up a more detailed description of that example along with all arguments that it accepts to configure its functionality.
<!--
#### Net
A detailed discussion of using Pilosa to index network traffic data is available [here - TODO]link blog post). This will discuss the implementation of `pdk net` as it relates to the use of the PDK library tools.
-->

121
docs/query-language.md
View file

@ -25,17 +25,17 @@ There will be one item in the `results` array for each PQL query in the request.
* Angle Brackets `<>` denote required arguments
* Square Brackets `[]` denote optional arguments
* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ROW_LABEL`, `STRING`)
* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ATTR_NAME`, `STRING`)
##### Examples
Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data.
The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would:
The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", col=10, row=1)` against a server using curl, you would:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", columnID=10, rowID=1)'
-d 'SetBit(frame="stargazer", col=10, row=1)'
```
``` response
{"results":[true]}
@ -58,7 +58,7 @@ curl localhost:10101/index/repository/query \
**Spec:**
```
SetBit(<frame=STRING>, <ROW_LABEL=UINT>, <COL_LABEL=UINT>,
SetBit(<frame=STRING>, <row=UINT>, <col=UINT>,
[timestamp=TIMESTAMP])
```
@ -76,26 +76,26 @@ A return value of `false` indicates that the bit was already set to 1 and nothin
**Examples:**
```
SetBit(frame="stargazer", columnID=10, rowID=1)
SetBit(frame="stargazer", col=10, row=1)
```
This query illustrates setting a bit in the stargazer frame. User with id=1 has starred repository with id=10.
SetBit also supports providing a timestamp. To write the date that a user starred a repository.
```
SetBit(frame="stargazer", columnID=10, rowID=1, timestamp="2016-01-01T00:00")
SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00")
```
Setting multiple bits in a single request:
```
SetBit(frame="stargazer", columnID=10, rowID=1) SetBit(frame="stargazer", columnID=10, rowID=2) SetBit(frame="stargazer", columnID=20, rowID=1) SetBit(frame="stargazer", columnID=30, rowID=2)
SetBit(frame="stargazer", col=10, row=1) SetBit(frame="stargazer", col=10, row=2) SetBit(frame="stargazer", col=20, row=1) SetBit(frame="stargazer", col=30, row=2)
```
#### SetRowAttrs
**Spec:**
```
SetRowAttrs(<frame=STRING>, <ROW_LABEL=UINT>,
SetRowAttrs(<frame=STRING>, <row=UINT>,
<ATTR_NAME=ATTR_VALUE>,
[ATTR_NAME=ATTR_VALUE ...])
```
@ -111,13 +111,13 @@ SetRowAttrs queries always return `null` upon success.
**Examples:**
```
SetRowAttrs(frame="stargazer", rowID=10, username="mrpi", active=true)
SetRowAttrs(frame="stargazer", row=10, username="mrpi", active=true)
```
Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", stargazer_id=10)`.
```
SetRowAttrs(frame="stargazer", rowID=10, username=null)
SetRowAttrs(frame="stargazer", row=10, username=null)
```
Delete username value for user 10.
@ -127,7 +127,7 @@ Delete username value for user 10.
**Spec:**
```
SetColumnAttrs(<frame=STRING>, <ROW_LABEL=UINT>,
SetColumnAttrs(<frame=STRING>, <row=UINT>,
<ATTR_NAME=ATTR_VALUE>,
[ATTR_NAME=ATTR_VALUE ...])
```
@ -143,13 +143,13 @@ SetColumnAttrs queries always return `null` upon success. Setting a value of `nu
**Examples:**
```
SetColumnAttrs(columnID=10, stars=123, url="http://projects.pilosa.com/10", active=true)
SetColumnAttrs(col=10, stars=123, url="http://projects.pilosa.com/10", active=true)
```
Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", columnID=10)`.
Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", col=10)`.
```
SetColumnAttrs(columnID=10, url=null)
SetColumnAttrs(col=10, url=null)
```
Delete url value for repo 10.
@ -160,7 +160,7 @@ Delete url value for repo 10.
**Spec:**
```
SetBit(<frame=STRING>, <ROW_LABEL=UINT>, <COL_LABEL=UINT>,
SetBit(<frame=STRING>, <row=UINT>, <col=UINT>,
[timestamp=TIMESTAMP])
```
@ -177,7 +177,7 @@ A return value of `false` indicates that the bit was already set to 0 and nothin
**Examples:**
```
ClearBit(frame="stargazer", columnID=10, rowID=1)
ClearBit(frame="stargazer", col=10, row=1)
```
Remove relationship between the stargazer in row 1 and the repository in column 10 from the stargazer frame.
@ -188,12 +188,12 @@ Remove relationship between the stargazer in row 1 and the repository in column
**Spec:**
```
SetFieldValue(<COL_LABEL=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
SetFieldValue(<col=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
```
**Description:**
`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`.
`SetFieldValue` assigns an integer value with the specified field name to the `col` in the given `frame`.
**Result Type:** null
@ -203,7 +203,7 @@ SetFieldValue returns `null` upon success.
Set the number of pull requests of repository 10.
```
SetFieldValue(columnID=10, frame="stats", pullrequests=2)
SetFieldValue(col=10, frame="stats", pullrequests=2)
```
@ -214,7 +214,7 @@ SetFieldValue(columnID=10, frame="stats", pullrequests=2)
**Spec:**
```
Bitmap(<frame=STRING>, (<ROW_LABEL=UINT> | <COL_LABEL>=UINT))
Bitmap(<frame=STRING>, (<rowL=UINT> | <col>=UINT))
```
**Description:**
@ -229,7 +229,7 @@ e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}`
Query all repositories that user 1 has starred.
```
Bitmap(frame="stargazer", rowID=1)
Bitmap(frame="stargazer", row=1)
```
Returns `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}`
@ -286,7 +286,7 @@ attrs will always be empty
Query repositories which have been starred by two users.
```
Intersect(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2))
Intersect(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2))
```
Returns `{"attrs":{},"bits":[10]}`.
@ -313,7 +313,7 @@ attrs will always be empty
Query repositories which have been starred by one user and not another.
```
Difference(Bitmap(frame="stargazer", rowID=1), Bitmap( frame="stargazer", rowID=2))
Difference(Bitmap(frame="stargazer", row=1), Bitmap( frame="stargazer", row=2))
```
Return `{"results":[{"attrs":{},"bits":[20]}]}`
@ -321,7 +321,7 @@ Return `{"results":[{"attrs":{},"bits":[20]}]}`
* bits are repositories that were starred by user 1 BUT NOT user 2
```
Difference(Bitmap(frame="stargazer", rowID=2), Bitmap( frame="stargazer", rowID=1))
Difference(Bitmap(frame="stargazer", row=2), Bitmap( frame="stargazer", row=1))
```
Return `{"attrs":{},"bits":[30]}`
@ -349,7 +349,7 @@ attrs will always be empty
Query repositories which have been starred by two users.
```
Xor(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2))
Xor(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2))
```
Returns `{"attrs":{},"bits":[30]}`.
@ -373,7 +373,7 @@ Returns the number of set bits in the `BITMAP_CALL` passed in.
Query the number of repositories to which a user has contributed.
```
Count(Bitmap(frame="stargazer", rowID=1))
Count(Bitmap(frame="stargazer", row=1))
```
Return `2`
@ -386,13 +386,12 @@ Return `2`
```
TopN([BITMAP_CALL], <frame=STRING>, [n=UINT],
[inverse=true], [<field=ATTR_NAME>, <filters=[]ATTR_VALUE>])
[<field=ATTR_NAME>, <filters=[]ATTR_VALUE>])
```
**Description:**
Return the id and count of the top `n` bitmaps (by count of bits) in the frame.
`inverse=true` specifies that the call should operate on the [inverse view ](../data-model/#inverse).
The `field` and `filters` arguments work together to only return Bitmaps which
have the attribute specified by `field` with one of the values specified in
`filters`.
@ -419,16 +418,6 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1}
* count is amount of repositories
* Results are the number of repositories that each user starred in descending order for all users in the stargazer frame, for example user 1 starred two repositories, user 2 starred two repositories, user 3 starred one repository.
```
TopN(frame="stargazer", inverse=true)
```
Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1}]`
* key is a repository ID
* count is amount of users
* Results are the number of users that starred each repository in descending order for all respositories in the stargazer frame.
```
TopN(frame="stargazer", n=2)
```
@ -438,7 +427,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}]`
* Results are the top two users sorted by number of repositories they've starred in descending order.
```
TopN(Bitmap(frame="language", rowID=1), frame="stargazer", n=2)
TopN(Bitmap(frame="language", row=1), frame="stargazer", n=2)
```
Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]`
@ -450,7 +439,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]`
**Spec:**
```
Range(<frame=STRING>, <ROW_LABEL=UINT>,
Range(<frame=STRING>, <row=UINT>,
<start=TIMESTAMP>, <end=TIMESTAMP>)
```
@ -466,7 +455,7 @@ between the given `start` and `end` timestamps.
When you set timestamp using SetBit, you will able to query all repositories that a user has starred within a date range.
```
Range(frame="stargazer", rowID=1, start="2010-01-01T00:00", end="2017-03-02T03:00")
Range(frame="stargazer", row=1, start="2010-01-01T00:00", end="2017-03-02T03:00")
```
Returns `{{"attrs":{},"bits":[10]}`
@ -523,6 +512,56 @@ Range(frame="stats", commitactivity >< [100, 200])
This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way.
#### Min
**Spec:**
```
Min([BITMAP_CALL], <frame=STRING>, <field=STRING>)
```
**Description:**
Returns the minimum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all collumns are considered.
**Result Type:** object with the min and count of columns containing the min value.
**Examples:**
Query the size of all repositories.
```
Min(frame="stats", field="diskusage")
```
Return `{"min":4,"count":2}`
* Result is the smallest repository in kilobytes, plus the number of repositories of that size.
#### Max
**Spec:**
```
Max([BITMAP_CALL], <frame=STRING>, <field=STRING>)
```
**Description:**
Returns the maximum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
**Result Type:** object with the max and count of columns containing the max value.
**Examples:**
Query the size of all repositories.
```
Max(frame="stats", field="diskusage")
```
Return `{"max":88,"count":13}`
* Result is the largest repository in kilobytes, plus the number of repositories of that size.
#### Sum
**Spec:**

137
docs/tutorials.md
View file

@ -70,6 +70,8 @@ We now should have `pilosa.local.gossip32` in the current directory with 32 rand
Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes.
One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. The coordinator is only important during cluster resizing operations, and otherwise acts like any other node in the cluster. In the future, the coordinator will be chosen transparently by distributed consensus, and this option will be deprecated.
Create `node1.config.toml` in the project directory and paste the following in it:
```toml
@ -79,7 +81,7 @@ data-dir = "node1_data"
bind = "https://01.pilosa.local:10501"
[cluster]
hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
coordinator = true
[tls]
certificate = "pilosa.local.crt"
@ -87,7 +89,7 @@ key = "pilosa.local.key"
skip-verify = true
[gossip]
seed = "01.pilosa.local:15000"
seeds = ["01.pilosa.local:15000"]
port = 15000
key = "pilosa.local.gossip32"
```
@ -100,16 +102,13 @@ Create `node2.config.toml` in the project directory and paste the following in i
data-dir = "node2_data"
bind = "https://02.pilosa.local:10502"
[cluster]
hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
[tls]
certificate = "pilosa.local.crt"
key = "pilosa.local.key"
skip-verify = true
[gossip]
seed = "01.pilosa.local:15000"
seeds = ["01.pilosa.local:15000"]
port = 16000
key = "pilosa.local.gossip32"
```
@ -122,16 +121,13 @@ Create `node3.config.toml` in the project directory and paste the following in i
data-dir = "node3_data"
bind = "https://03.pilosa.local:10503"
[cluster]
hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
[tls]
certificate = "pilosa.local.crt"
key = "pilosa.local.key"
skip-verify = true
[gossip]
seed = "01.pilosa.local:15000"
seeds = ["01.pilosa.local:15000"]
port = 17000
key = "pilosa.local.gossip32"
```
@ -140,9 +136,9 @@ Here is some explanation of the configuration items:
* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it.
* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes.
* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](../configuration/#cluster-hosts) for other settings.
* `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-coordinator) for other settings.
* `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers.
* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before.
* `[gossip]` section contains settings for the Gossip protocol. `seeds` contain the seed nodes which other nodes gather cluster topology. There must be at least one gossip seed. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before.
#### Final Touches Before Running the Cluster
@ -189,9 +185,9 @@ curl -k --ipv4 https://01.pilosa.local:10501/status
The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL.
All nodes should be in the `UP` state:
All nodes should be in the `NORMAL` state:
``` response
{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}}
{"state":"NORMAL","nodes":[{"id":"98ebd177-c082-4c54-8d48-7e7c75857b52","uri":{"scheme":"https","host":"02.pilosa.local","port":10502},"isCoordinator":false},{"id":"a33dc0d6-c35f-4559-984a-e582bf032a21","uri":{"scheme":"https","host":"03.pilosa.local","port":10503},"isCoordinator":false},{"id":"e24ac014-ee2f-4cb0-b565-74df6c551f0a","uri":{"scheme":"https","host":"01.pilosa.local","port":10501},"isCoordinator":true}]}
```
#### Running Queries
@ -216,7 +212,7 @@ We just created frame `sample-frame` with default options.
Let's run a `SetBit` query:
``` request
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)'
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", row=1, col=100)'
```
``` response
{"results":[true]}
@ -224,7 +220,7 @@ curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit
Confirm that the bit was indeed set:
``` request
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)'
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)'
```
``` response
{"results":[{"attrs":{},"bits":[100]}]}
@ -232,7 +228,7 @@ curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap
The same response should be returned when querying other nodes in the cluster:
``` request
curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)'
curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)'
```
``` response
{"results":[{"attrs":{},"bits":[100]}]}
@ -247,7 +243,7 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ
#### Introduction
Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range` and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
First, create an index called `patients`:
``` request
@ -263,7 +259,6 @@ In addition to storing rows of bits, a frame can also contain fields that store
curl localhost:10101/index/patients/frame/measurements \
-X POST \
-d '{"options":{
"rangeEnabled": true,
"fields": [
{"name": "age", "type": "int", "min": 0, "max": 120},
{"name": "weight", "type": "int", "min": 0, "max": 500},
@ -283,7 +278,7 @@ This query sets the age, weight, and t-cell count for the patient with ID `1` in
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'SetFieldValue(columnID=1, frame="measurements", age=34, weight=128, tcells=1145)'
-d 'SetFieldValue(col=1, frame="measurements", age=34, weight=128, tcells=1145)'
```
``` response
{"results":[null]}
@ -333,7 +328,7 @@ curl localhost:10101/index/patients/query \
```
The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`.
You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40.
You can also provide a filter to the `Sum()` function to find the average age of all patients over 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
@ -344,6 +339,48 @@ curl localhost:10101/index/patients/query \
```
Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query.
To find the minimum age of all patients, run a `Min` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Min(frame="measurements", field="age")'
```
``` response
{"results":[{"min":19,"count":1}]}
```
The results you get from the `Min` query contain the `min` of all values as well as the `count` of columns with that value.
You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Min(Range(frame="measurements", age > 40), frame="measurements", field="age")'
```
``` response
{"results":[{"min":57,"count":1}]}
```
To find the maximum age of all patients, run a `Max` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Max(frame="measurements", field="age")'
```
``` response
{"results":[{"max":71,"count":1}]}
```
The results you get from the `Max` query contain the `max` of all values as well as the `count` of columns with that value.
You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Max(Range(frame="measurements", age < 40), frame="measurements", field="age")'
```
``` response
{"results":[{"max":34,"count":1}]}
```
### Storing Row and Column Attributes
#### Introduction
@ -373,11 +410,11 @@ Now, let's add some books to our index.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetColumnAttrs(columnID=1, name="To Kill a Mockingbird", year=1960)
SetColumnAttrs(columnID=2, name="No Name in the Street", year=1972)
SetColumnAttrs(columnID=3, name="The Tipping Point", year=2000)
SetColumnAttrs(columnID=4, name="Out Stealing Horses", year=2003)
SetColumnAttrs(columnID=5, name="The Forever War", year=2008)'
-d 'SetColumnAttrs(col=1, name="To Kill a Mockingbird", year=1960)
SetColumnAttrs(col=2, name="No Name in the Street", year=1972)
SetColumnAttrs(col=3, name="The Tipping Point", year=2000)
SetColumnAttrs(col=4, name="Out Stealing Horses", year=2003)
SetColumnAttrs(col=5, name="The Forever War", year=2008)'
```
``` response
{"results":[null,null,null,null,null]}
@ -387,11 +424,11 @@ And add some members.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetRowAttrs(frame="members", rowID=10001, fullName="John Smith")
SetRowAttrs(frame="members", rowID=10002, fullName="Sue Perkins")
SetRowAttrs(frame="members", rowID=10003, fullName="Jennifer Hawks")
SetRowAttrs(frame="members", rowID=10004, fullName="Pedro Vazquez")
SetRowAttrs(frame="members", rowID=10005, fullName="Pat Washington")'
-d 'SetRowAttrs(frame="members", row=10001, fullName="John Smith")
SetRowAttrs(frame="members", row=10002, fullName="Sue Perkins")
SetRowAttrs(frame="members", row=10003, fullName="Jennifer Hawks")
SetRowAttrs(frame="members", row=10004, fullName="Pedro Vazquez")
SetRowAttrs(frame="members", row=10005, fullName="Pat Washington")'
```
``` response
{"results":[null,null,null,null,null]}
@ -401,7 +438,7 @@ At this point we can query one of the `member` records by querying that row.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
-d 'Bitmap(frame="members", row=10002)'
```
``` response
{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]}
@ -411,23 +448,19 @@ Now let's add some data to the matrix such that each pair represents a member wh
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetBit(frame="members", rowID=10001, columnID=3)
SetBit(frame="members", rowID=10001, columnID=5)
SetBit(frame="members", rowID=10002, columnID=1)
SetBit(frame="members", rowID=10002, columnID=2)
SetBit(frame="members", rowID=10002, columnID=4)
SetBit(frame="members", rowID=10003, columnID=3)
SetBit(frame="members", rowID=10004, columnID=4)
SetBit(frame="members", rowID=10004, columnID=5)
SetBit(frame="members", rowID=10005, columnID=1)
SetBit(frame="members", rowID=10005, columnID=2)
SetBit(frame="members", rowID=10005, columnID=3)
SetBit(frame="members", rowID=10005, columnID=4)
SetBit(frame="members", rowID=10005, columnID=5)'
-d 'SetBit(frame="members", row=10001, col=3)
SetBit(frame="members", row=10001, col=5)
SetBit(frame="members", row=10002, col=1)
SetBit(frame="members", row=10002, col=2)
SetBit(frame="members", row=10002, col=4)
SetBit(frame="members", row=10003, col=3)
SetBit(frame="members", row=10004, col=4)
SetBit(frame="members", row=10004, col=5)
SetBit(frame="members", row=10005, col=1)
SetBit(frame="members", row=10005, col=2)
SetBit(frame="members", row=10005, col=3)
SetBit(frame="members", row=10005, col=4)
SetBit(frame="members", row=10005, col=5)'
```
``` response
{"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]}
@ -437,7 +470,7 @@ Now pull the record for `Sue Perkins` again.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
-d 'Bitmap(frame="members", row=10002)'
```
``` response
{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]}
@ -448,7 +481,7 @@ In order to retrieve the attribute information that we stored for each book, we
``` request
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
-d 'Bitmap(frame="members", row=10002)'
```
``` response
{
@ -466,7 +499,7 @@ Finally, if we want to find out which books were read by both `Sue` and `Pedro`,
``` request
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Intersect(Bitmap(frame="members", rowID=10002), Bitmap(frame="members", rowID=10004))'
-d 'Intersect(Bitmap(frame="members", row=10002), Bitmap(frame="members", row=10004))'
```
``` response
{

4
docs/webui.md
View file

@ -32,9 +32,9 @@ In addition to standard PQL, the console supports a few special commands, prefix
- `:create frame <framename>`
- `:delete frame <framename>`
Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame).
Frame creation also supports options like `timeQuantum`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame).
- `:create frame <framename> inverseEnabled=true cacheSize=10000`
- `:create frame <framename> cacheSize=10000`
### Cluster Admin

300
executor.go
View file

@ -84,29 +84,31 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// to send queries to different slices based on orientation.
var inverseSlices []uint64
// If slices aren't specified, then include all of them.
if len(slices) == 0 {
// Determine slices and inverseSlices for use in e.executeCall().
if needsSlices {
// Round up the number of slices.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
maxSlice := idx.MaxSlice()
maxInverseSlice := idx.MaxInverseSlice()
// If slices are specified, then use that value for slices or
// inverseSlices. If slices aren't specified, then include all of them.
if len(slices) > 0 {
// For inverse queries, the values of `slices` provided to the Execute() method
// on the remote node actually represents inverseSlices.
inverseSlices = slices
} else if needsSlices {
// Round up the number of slices.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
maxSlice := idx.MaxSlice()
maxInverseSlice := idx.MaxInverseSlice()
// Generate a slices of all slices.
slices = make([]uint64, maxSlice+1)
for i := range slices {
slices[i] = uint64(i)
}
// Generate a slices of all slices.
slices = make([]uint64, maxSlice+1)
for i := range slices {
slices[i] = uint64(i)
}
// Generate a slices of all inverse slices.
inverseSlices = make([]uint64, maxInverseSlice+1)
for i := range inverseSlices {
inverseSlices[i] = uint64(i)
}
// Generate a slices of all inverse slices.
inverseSlices = make([]uint64, maxInverseSlice+1)
for i := range inverseSlices {
inverseSlices[i] = uint64(i)
}
}
@ -118,7 +120,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for _, call := range q.Calls {
if call.SupportsInverse() && needsSlices {
// Fetch frame & row label based on argument.
frame, _ := call.Args["frame"].(string)
@ -147,7 +148,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// executeCall executes a call.
func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) {
if err := e.validateCallArgs(c); err != nil {
return nil, err
}
@ -157,6 +157,12 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
case "Sum":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeSum(ctx, index, c, slices, opt)
case "Min":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeFieldMin(ctx, index, c, slices, opt)
case "Max":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeFieldMax(ctx, index, c, slices, opt)
case "ClearBit":
return e.executeClearBit(ctx, index, c, opt)
case "Count":
@ -199,15 +205,15 @@ func (e *Executor) validateCallArgs(c *pql.Call) error {
}
// executeSum executes a Sum() call.
func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (SumCount, error) {
func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return SumCount{}, errors.New("Sum(): frame required")
return ValCount{}, errors.New("Sum(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return SumCount{}, errors.New("Sum(): field required")
return ValCount{}, errors.New("Sum(): field required")
}
if len(c.Children) > 1 {
return SumCount{}, errors.New("Sum() only accepts a single bitmap input")
return ValCount{}, errors.New("Sum() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
@ -217,18 +223,88 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(SumCount)
return other.Add(v.(SumCount))
other, _ := prev.(ValCount)
return other.Add(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return SumCount{}, err
return ValCount{}, err
}
other, _ := result.(SumCount)
other, _ := result.(ValCount)
if other.Count == 0 {
return SumCount{}, nil
return ValCount{}, nil
}
return other, nil
}
// executeFieldMin executes a Min() call.
func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return ValCount{}, errors.New("Min(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Min(): field required")
}
if len(c.Children) > 1 {
return ValCount{}, errors.New("Min() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeFieldMinSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.Smaller(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return ValCount{}, err
}
other, _ := result.(ValCount)
if other.Count == 0 {
return ValCount{}, nil
}
return other, nil
}
// executeFieldMax executes a Max() call.
func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return ValCount{}, errors.New("Max(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Max(): field required")
}
if len(c.Children) > 1 {
return ValCount{}, errors.New("Max() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeFieldMaxSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.Larger(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return ValCount{}, err
}
other, _ := result.(ValCount)
if other.Count == 0 {
return ValCount{}, nil
}
return other, nil
}
@ -318,13 +394,13 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *
}
}
// executeSumCountSlice executes calculates the sum & count for fields on a slice.
func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) {
// executeSumCountSlice calculates the sum and count for fields on a slice.
func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return SumCount{}, err
return ValCount{}, err
}
filter = bm
}
@ -334,29 +410,107 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return SumCount{}, nil
return ValCount{}, nil
}
field := frame.Field(fieldName)
if field == nil {
return SumCount{}, nil
return ValCount{}, nil
}
view := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if view == nil {
return SumCount{}, nil
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
vsum, vcount, err := view.FieldSum(filter, field.BitDepth())
vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth())
if err != nil {
return SumCount{}, err
return ValCount{}, err
}
return SumCount{
Sum: int64(vsum) + (int64(vcount) * field.Min),
return ValCount{
Val: int64(vsum) + (int64(vcount) * field.Min),
Count: int64(vcount),
}, nil
}
// executeFieldMinSlice calculates the min for fields on a slice.
func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return ValCount{}, err
}
filter = bm
}
frameName, _ := c.Args["frame"].(string)
fieldName, _ := c.Args["field"].(string)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ValCount{}, nil
}
field := frame.Field(fieldName)
if field == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth())
if err != nil {
return ValCount{}, err
}
return ValCount{
Val: int64(fmin) + field.Min,
Count: int64(fcount),
}, nil
}
// executeFieldMaxSlice calculates the max for fields on a slice.
func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return ValCount{}, err
}
filter = bm
}
frameName, _ := c.Args["frame"].(string)
fieldName, _ := c.Args["field"].(string)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ValCount{}, nil
}
field := frame.Field(fieldName)
if field == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth())
if err != nil {
return ValCount{}, err
}
return ValCount{
Val: int64(fmax) + field.Min,
Count: int64(fcount),
}, nil
}
// executeTopN executes a TopN() call.
// This first performs the TopN() to determine the top results and then
// requeries to retrieve the full counts for each of the top results.
@ -941,7 +1095,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.FragmentNodes(index, slice) {
for _, node := range e.Cluster.SliceNodes(index, slice) {
// Update locally if host matches.
if node.ID == e.Node.ID {
val, err := f.ClearBit(view, rowID, colID, nil)
@ -1042,7 +1196,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.FragmentNodes(index, slice) {
for _, node := range e.Cluster.SliceNodes(index, slice) {
// Update locally if host matches.
if node.ID == e.Node.ID {
val, err := f.SetBit(view, rowID, colID, timestamp)
@ -1355,7 +1509,7 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *
switch call.Name {
case "Average", "Sum":
v, err = decodeSumCount(pb.Results[i].GetSumCount()), nil
v, err = decodeValCount(pb.Results[i].GetValCount()), nil
case "TopN":
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case "Count":
@ -1385,7 +1539,7 @@ func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (m
loop:
for _, slice := range slices {
for _, node := range e.Cluster.FragmentNodes(index, slice) {
for _, node := range e.Cluster.SliceNodes(index, slice) {
if Nodes(nodes).Contains(node) {
m[node] = append(m[node], slice)
continue loop
@ -1593,29 +1747,51 @@ func needsSlices(calls []*pql.Call) bool {
return false
}
// SumCount represents a grouping of sum & count for Sum() and Average() calls.
type SumCount struct {
Sum int64 `json:"sum"`
// ValCount represents a grouping of sum & count for Sum() and Average() calls.
type ValCount struct {
Val int64 `json:"value"`
Count int64 `json:"count"`
}
func (sc *SumCount) Add(other SumCount) SumCount {
return SumCount{
Sum: sc.Sum + other.Sum,
Count: sc.Count + other.Count,
func (vc *ValCount) Add(other ValCount) ValCount {
return ValCount{
Val: vc.Val + other.Val,
Count: vc.Count + other.Count,
}
}
func encodeSumCount(sc SumCount) *internal.SumCount {
return &internal.SumCount{
Sum: sc.Sum,
Count: sc.Count,
func encodeValCount(vc ValCount) *internal.ValCount {
return &internal.ValCount{
Val: vc.Val,
Count: vc.Count,
}
}
func decodeSumCount(pb *internal.SumCount) SumCount {
return SumCount{
Sum: pb.Sum,
func decodeValCount(pb *internal.ValCount) ValCount {
return ValCount{
Val: pb.Val,
Count: pb.Count,
}
}
// Smaller returns the smaller of the two ValCounts.
func (vc *ValCount) Smaller(other ValCount) ValCount {
if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) {
return other
}
return ValCount{
Val: vc.Val,
Count: vc.Count,
}
}
// Larger returns the larger of the two ValCounts.
func (vc *ValCount) Larger(other ValCount) ValCount {
if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) {
return other
}
return ValCount{
Val: vc.Val,
Count: vc.Count,
}
}

115
executor_test.go
View file

@ -280,7 +280,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
// Create frames.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 50},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 1, Max: 2},
@ -330,7 +329,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 100},
},
@ -599,8 +597,8 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
}
}
// Ensure a Sum() query can be executed.
func TestExecutor_Execute_Sum(t *testing.T) {
// Ensure Min() and Max() queries can be executed.
func TestExecutor_Execute_MinMax(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
@ -612,6 +610,97 @@ func TestExecutor_Execute_Sum(t *testing.T) {
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100},
},
}); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=3)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(frame=f, row=1, col=1)
SetBit(frame=f, row=2, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, col=0)
SetFieldValue(frame=f, foo=-5, col=1)
SetFieldValue(frame=f, foo=-5, col=2)
SetFieldValue(frame=f, foo=10, col=3)
SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("Min", func(t *testing.T) {
tests := []struct {
filter string
exp int64
cnt int64
}{
{filter: ``, exp: -5, cnt: 2},
{filter: `Bitmap(frame=f, row=0)`, exp: 10, cnt: 1},
{filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
if tt.filter == "" {
pql = `Min(frame=f, field=foo)`
} else {
pql = fmt.Sprintf(`Min(%s, frame=f, field=foo)`, tt.filter)
}
if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) {
t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result))
}
}
})
t.Run("Max", func(t *testing.T) {
tests := []struct {
filter string
exp int64
cnt int64
}{
{filter: ``, exp: 60, cnt: 1},
{filter: `Bitmap(frame=f, row=0)`, exp: 60, cnt: 1},
{filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
if tt.filter == "" {
pql = `Max(frame=f, field=foo)`
} else {
pql = fmt.Sprintf(`Max(%s, frame=f, field=foo)`, tt.filter)
}
if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) {
t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result))
}
}
})
}
// Ensure a Sum() query can be executed.
func TestExecutor_Execute_Sum(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
@ -621,7 +710,6 @@ func TestExecutor_Execute_Sum(t *testing.T) {
}
if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
},
@ -646,7 +734,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
t.Run("NoFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 200, Count: 5}) {
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -654,7 +742,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
t.Run("WithFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 80, Count: 2}) {
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -723,7 +811,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
@ -733,7 +820,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
}
if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
},
@ -742,7 +828,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
}
if _, err := idx.CreateFrame("edge", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: -100, Max: 100},
},
@ -927,7 +1012,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
// The local node owns slice 1.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
e := test.NewExecutor(hldr.Holder, c)
@ -961,7 +1046,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Create local executor data. The local node owns slice 1.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2)
@ -1004,7 +1089,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Create local executor data.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
// Create frame.
if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
@ -1056,7 +1141,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Create local executor data.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
// Create frame.
if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
@ -1130,7 +1215,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Create local executor data on slice 2 & 4.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)

78
fragment.go
View file

@ -614,6 +614,70 @@ func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, e
return sum, count, nil
}
// FieldMin returns the min of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
// If there are no columns to consider, return early.
if consider.Count() == 0 {
return 0, 0, nil
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
row := f.Row(uint64(ii))
x := consider.Difference(row)
count = x.Count()
if count > 0 {
consider = x
} else {
min += (1 << ii)
if ii == 0 {
count = consider.Count()
}
}
}
return min, count, nil
}
// FieldMax returns the max of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
// If there are no columns to consider, return early.
if consider.Count() == 0 {
return 0, 0, nil
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
row := f.Row(uint64(ii))
x := row.Intersect(consider)
count = x.Count()
if count > 0 {
max += (1 << ii)
consider = x
} else if ii == 0 {
count = consider.Count()
}
}
return max, count, nil
}
// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) {
switch op {
@ -1702,7 +1766,7 @@ func (s *FragmentSyncer) isClosing() bool {
// then merges any blocks which have differences.
func (s *FragmentSyncer) SyncFragment() error {
// Determine replica set.
nodes := s.Cluster.FragmentNodes(s.Fragment.Index(), s.Fragment.Slice())
nodes := s.Cluster.SliceNodes(s.Fragment.Index(), s.Fragment.Slice())
if len(nodes) == 1 {
return nil
}
@ -1784,7 +1848,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Read pairs from each remote block.
var pairSets []PairSet
var clients []InternalClient
for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) {
for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) {
if s.Node.ID == node.ID {
continue
}
@ -1832,15 +1896,19 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest.
total := len(set.ColumnIDs) + len(clear.ColumnIDs)
buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(s.Cluster.MaxWritesPerRequest))))
maxWrites := s.Cluster.MaxWritesPerRequest
if maxWrites <= 0 {
maxWrites = 5000
}
buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites))))
// Only sync the standard block.
for j := 0; j < len(set.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
count++
}
for j := 0; j < len(clear.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
count++
}

73
fragment_test.go
View file

@ -256,6 +256,79 @@ func TestFragment_FieldSum(t *testing.T) {
})
}
// Ensure a fragment can find the min and max of field values.
func TestFragment_FieldMinMax(t *testing.T) {
const bitDepth = 16
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil {
t.Fatal(err)
}
t.Run("Min", func(t *testing.T) {
tests := []struct {
filter *pilosa.Bitmap
exp uint64
cnt uint64
}{
{filter: nil, exp: 0, cnt: 1},
{filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 300, cnt: 2},
{filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2},
{filter: pilosa.NewBitmap(1), exp: 0, cnt: 0},
{filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1},
{filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1},
}
for i, test := range tests {
if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil {
t.Fatal(err)
} else if min != test.exp {
t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min)
} else if cnt != test.cnt {
t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt)
}
}
})
t.Run("Max", func(t *testing.T) {
tests := []struct {
filter *pilosa.Bitmap
exp uint64
cnt uint64
}{
{filter: nil, exp: 2818, cnt: 2},
{filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 2818, cnt: 1},
{filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2},
{filter: pilosa.NewBitmap(1), exp: 0, cnt: 0},
{filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1},
{filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1},
}
for i, test := range tests {
if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil {
t.Fatal(err)
} else if max != test.exp {
t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max)
} else if cnt != test.cnt {
t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt)
}
}
})
}
// Ensure a fragment query for matching fields.
func TestFragment_FieldRange(t *testing.T) {
const bitDepth = 16

75
frame.go
View file

@ -33,7 +33,6 @@ import (
const (
DefaultCacheType = CacheTypeRanked
DefaultInverseEnabled = false
DefaultRangeEnabled = false
// Default ranked frame cache
DefaultCacheSize = 50000
@ -59,7 +58,6 @@ type Frame struct {
cacheType string
cacheSize uint32
timeQuantum TimeQuantum
rangeEnabled bool
fields []*Field
Logger Logger
@ -88,7 +86,6 @@ func NewFrame(path, index, name string) (*Frame, error) {
cacheType: DefaultCacheType,
cacheSize: DefaultCacheSize,
//timeQuantum
rangeEnabled: DefaultRangeEnabled,
//fields
Logger: NopLogger,
@ -145,11 +142,6 @@ func (f *Frame) InverseEnabled() bool {
return f.inverseEnabled
}
// RangeEnabled returns true if range fields can be stored on this frame.
func (f *Frame) RangeEnabled() bool {
return f.rangeEnabled
}
// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update.
// defaults to DefaultCacheSize 50000
func (f *Frame) SetCacheSize(v uint32) error {
@ -188,7 +180,6 @@ func (f *Frame) Options() FrameOptions {
func (f *Frame) options() FrameOptions {
return FrameOptions{
InverseEnabled: f.inverseEnabled,
RangeEnabled: f.rangeEnabled,
CacheType: f.cacheType,
CacheSize: f.cacheSize,
TimeQuantum: f.timeQuantum,
@ -268,7 +259,6 @@ func (f *Frame) loadMeta() error {
f.cacheType = DefaultCacheType
f.cacheSize = DefaultCacheSize
f.timeQuantum = ""
f.rangeEnabled = DefaultRangeEnabled
//f.fields
return nil
} else if err != nil {
@ -287,7 +277,6 @@ func (f *Frame) loadMeta() error {
}
f.cacheSize = pb.CacheSize
f.timeQuantum = TimeQuantum(pb.TimeQuantum)
f.rangeEnabled = pb.RangeEnabled
f.fields = decodeFields(pb.Fields)
return nil
@ -365,11 +354,6 @@ func (f *Frame) CreateField(field *Field) error {
f.mu.Lock()
defer f.mu.Unlock()
// Ensure frame supports fields.
if !f.RangeEnabled() {
return ErrFrameFieldsNotAllowed
}
// Append field.
if err := f.addField(field); err != nil {
return err
@ -402,11 +386,6 @@ func (f *Frame) GetFields() ([]*Field, error) {
f.mu.RLock()
defer f.mu.RUnlock()
// Ensure the frame supports fields.
if !f.RangeEnabled() {
return nil, ErrFrameFieldsNotAllowed
}
err := f.loadMeta()
if err != nil {
return nil, err
@ -420,11 +399,6 @@ func (f *Frame) DeleteField(name string) error {
f.mu.Lock()
defer f.mu.Unlock()
// Ensure frame supports fields.
if !f.RangeEnabled() {
return ErrFrameFieldsNotAllowed
}
// Remove field.
if err := f.deleteField(name); err != nil {
return err
@ -763,6 +737,46 @@ func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err err
return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil
}
// FieldMin returns the min for a field.
// An optional filtering bitmap can be provided.
func (f *Frame) FieldMin(filter *Bitmap, name string) (min, count int64, err error) {
field := f.Field(name)
if field == nil {
return 0, 0, ErrFieldNotFound
}
view := f.View(ViewFieldPrefix + name)
if view == nil {
return 0, 0, nil
}
vmin, vcount, err := view.FieldMin(filter, field.BitDepth())
if err != nil {
return 0, 0, err
}
return int64(vmin) + field.Min, int64(vcount), nil
}
// FieldMax returns the max for a field.
// An optional filtering bitmap can be provided.
func (f *Frame) FieldMax(filter *Bitmap, name string) (max, count int64, err error) {
field := f.Field(name)
if field == nil {
return 0, 0, ErrFieldNotFound
}
view := f.View(ViewFieldPrefix + name)
if view == nil {
return 0, 0, nil
}
vmax, vcount, err := view.FieldMax(filter, field.BitDepth())
if err != nil {
return 0, 0, err
}
return int64(vmax) + field.Min, int64(vcount), nil
}
func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap, error) {
// Retrieve and validate field.
field := f.Field(name)
@ -890,11 +904,6 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
// ImportValue bulk imports range-encoded value data.
func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error {
// Verify that this frame is range-encoded.
if !f.RangeEnabled() {
return fmt.Errorf("Frame not RangeEnabled: %s", f.name)
}
viewName := ViewFieldPrefix + fieldName
// Get the field so we know bitDepth.
field := f.Field(fieldName)
@ -991,7 +1000,7 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FrameOptions represents options to set when initializing a frame.
type FrameOptions struct {
InverseEnabled bool `json:"inverseEnabled,omitempty"`
RangeEnabled bool `json:"rangeEnabled,omitempty"`
RangeEnabled bool `json:"rangeEnabled,omitempty"` // deprecated, will be removed
CacheType string `json:"cacheType,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
@ -1009,7 +1018,6 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta {
}
return &internal.FrameMeta{
InverseEnabled: o.InverseEnabled,
RangeEnabled: o.RangeEnabled,
CacheType: o.CacheType,
CacheSize: o.CacheSize,
TimeQuantum: string(o.TimeQuantum),
@ -1023,7 +1031,6 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions {
}
return &FrameOptions{
InverseEnabled: options.InverseEnabled,
RangeEnabled: options.RangeEnabled,
CacheType: options.CacheType,
CacheSize: options.CacheSize,
TimeQuantum: TimeQuantum(options.TimeQuantum),

5
frame_test.go
View file

@ -77,7 +77,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25},
@ -123,7 +122,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
},
@ -161,7 +159,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
},
@ -181,7 +178,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
},
@ -201,7 +197,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
},

120
gossip/gossip.go
View file

@ -29,6 +29,7 @@ import (
"github.com/hashicorp/memberlist"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/toml"
"github.com/pkg/errors"
)
@ -151,7 +152,7 @@ type gossipConfig struct {
type GossipMemberSetOption func(*GossipMemberSet) error
// WithTransport is a functional option for providing a transport to NewGossipMemberSet.
func WithTransport(transport *Transport) func(*GossipMemberSet) error {
func WithTransport(transport *Transport) GossipMemberSetOption {
return func(g *GossipMemberSet) error {
g.transport = transport
return nil
@ -159,7 +160,7 @@ func WithTransport(transport *Transport) func(*GossipMemberSet) error {
}
// WithLogger is a functional option for providing a logger to NewGossipMemberSet.
func WithLogger(logger *log.Logger) func(*GossipMemberSet) error {
func WithLogger(logger *log.Logger) GossipMemberSetOption {
return func(g *GossipMemberSet) error {
g.logger = logger
return nil
@ -167,11 +168,8 @@ func WithLogger(logger *log.Logger) func(*GossipMemberSet) error {
}
// NewGossipMemberSet returns a new instance of GossipMemberSet based on options.
func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
g := &GossipMemberSet{
Logger: server.Logger,
}
func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventReceiver, sh pilosa.StatusHandler, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
g := &GossipMemberSet{}
// options
for _, opt := range options {
@ -181,17 +179,11 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server,
}
if g.transport == nil {
port, err := strconv.Atoi(cfg.Gossip.Port)
port, err := strconv.Atoi(cfg.Port)
if err != nil {
return nil, fmt.Errorf("convert port: %s", err)
}
bindURI, err := pilosa.NewURIFromAddress(cfg.Bind)
if err != nil {
return nil, fmt.Errorf("getting uri from bind address: %s", err)
}
host := bindURI.Host()
// Set up the transport.
transport, err := NewTransport(host, port, g.logger)
if err != nil {
@ -203,15 +195,10 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server,
port := g.transport.Net.GetAutoBindPort()
bindURI, err := pilosa.NewURIFromAddress(cfg.Bind)
if err != nil {
return nil, fmt.Errorf("getting uri from bind address (with transport): %s", err)
}
host := bindURI.Host()
var gossipKey []byte
if cfg.Gossip.Key != "" {
gossipKey, err = ioutil.ReadFile(cfg.Gossip.Key)
var err error
if cfg.Key != "" {
gossipKey, err = ioutil.ReadFile(cfg.Key)
if err != nil {
return nil, fmt.Errorf("reading gossip key: %s", err)
}
@ -226,26 +213,26 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server,
conf.AdvertisePort = port
conf.AdvertiseAddr = pilosa.HostToIP(host)
//
conf.TCPTimeout = time.Duration(cfg.Gossip.StreamTimeout)
conf.SuspicionMult = cfg.Gossip.SuspicionMult
conf.PushPullInterval = time.Duration(cfg.Gossip.PushPullInterval)
conf.ProbeTimeout = time.Duration(cfg.Gossip.ProbeTimeout)
conf.ProbeInterval = time.Duration(cfg.Gossip.ProbeInterval)
conf.GossipNodes = cfg.Gossip.Nodes
conf.GossipInterval = time.Duration(cfg.Gossip.Interval)
conf.GossipToTheDeadTime = time.Duration(cfg.Gossip.ToTheDeadTime)
conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
conf.SuspicionMult = cfg.SuspicionMult
conf.PushPullInterval = time.Duration(cfg.PushPullInterval)
conf.ProbeTimeout = time.Duration(cfg.ProbeTimeout)
conf.ProbeInterval = time.Duration(cfg.ProbeInterval)
conf.GossipNodes = cfg.Nodes
conf.GossipInterval = time.Duration(cfg.Interval)
conf.GossipToTheDeadTime = time.Duration(cfg.ToTheDeadTime)
//
conf.Delegate = g
conf.SecretKey = gossipKey
conf.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate)
conf.Events = ger
conf.Logger = g.logger
g.config = &gossipConfig{
memberlistConfig: conf,
gossipSeeds: cfg.Gossip.Seeds,
gossipSeeds: cfg.Seeds,
}
g.statusHandler = server
g.statusHandler = sh
return g, nil
}
@ -456,7 +443,7 @@ type Transport struct {
// NewTransport returns a NetTransport based on the given host and port.
// It will dynamically bind to a port if port is 0.
// This is useful for test cases where specifiying a port is not reasonable.
// This is useful for test cases where specifying a port is not reasonable.
//func NewTransport(host string, port int) (*memberlist.NetTransport, error) {
func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) {
// memberlist config
@ -526,3 +513,68 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
return nt, nil
}
// Config holds toml-friendly memberlist configuration.
type Config struct {
// Port indicates the port to which pilosa should bind for internal state sharing.
Port string `toml:"port"`
Seeds []string `toml:"seeds"`
Key string `toml:"key"`
// StreamTimeout is the timeout for establishing a stream connection with
// a remote node for a full state sync, and for stream read and write
// operations. Maps to memberlist TCPTimeout.
StreamTimeout toml.Duration `toml:"stream-timeout"`
// SuspicionMult is the multiplier for determining the time an
// inaccessible node is considered suspect before declaring it dead.
// The actual timeout is calculated using the formula:
//
// SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
//
// This allows the timeout to scale properly with expected propagation
// delay with a larger cluster size. The higher the multiplier, the longer
// an inaccessible node is considered part of the cluster before declaring
// it dead, giving that suspect node more time to refute if it is indeed
// still alive.
SuspicionMult int `toml:"suspicion-mult"`
// PushPullInterval is the interval between complete state syncs.
// Complete state syncs are done with a single node over TCP and are
// quite expensive relative to standard gossiped messages. Setting this
// to zero will disable state push/pull syncs completely.
//
// Setting this interval lower (more frequent) will increase convergence
// speeds across larger clusters at the expense of increased bandwidth
// usage.
PushPullInterval toml.Duration `toml:"push-pull-interval"`
// ProbeInterval and ProbeTimeout are used to configure probing behavior
// for memberlist.
//
// ProbeInterval is the interval between random node probes. Setting
// this lower (more frequent) will cause the memberlist cluster to detect
// failed nodes more quickly at the expense of increased bandwidth usage.
//
// ProbeTimeout is the timeout to wait for an ack from a probed node
// before assuming it is unhealthy. This should be set to 99-percentile
// of RTT (round-trip time) on your network.
ProbeInterval toml.Duration `toml:"probe-interval"`
ProbeTimeout toml.Duration `toml:"probe-timeout"`
// Interval and Nodes are used to configure the gossip
// behavior of memberlist.
//
// Interval is the interval between sending messages that need
// to be gossiped that haven't been able to piggyback on probing messages.
// If this is set to zero, non-piggyback gossip is disabled. By lowering
// this value (more frequent) gossip messages are propagated across
// the cluster more quickly at the expense of increased bandwidth.
//
// Nodes is the number of random nodes to send gossip messages to
// per Interval. Increasing this number causes the gossip messages
// to propagate across the cluster more quickly at the expense of
// increased bandwidth.
//
// ToTheDeadTime is the interval after which a node has died that
// we will still try to gossip to it. This gives it a chance to refute.
Interval toml.Duration `toml:"interval"`
Nodes int `toml:"nodes"`
ToTheDeadTime toml.Duration `toml:"to-the-dead-time"`
}

1219
handler.go
View file

File diff suppressed because it is too large Load diff

240
handler_test.go
View file

@ -65,8 +65,8 @@ func TestHandler_NotFound(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil))
@ -100,8 +100,8 @@ func TestHandler_Schema(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != http.StatusOK {
@ -139,16 +139,17 @@ func TestHandler_Status(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.StatusHandler = s
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Cluster.SetState(pilosa.ClusterStateNormal)
h.API.StatusHandler = s
s.Handler = h
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"test-node","uri":{"scheme":"http","host":"localhost","port":10101},"isCoordinator":false}]}`+"\n" {
} else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -158,14 +159,15 @@ func TestHandler_ClusterResizeAbort(t *testing.T) {
t.Run("No resize job", func(t *testing.T) {
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.SetRestricted()
h.API.Cluster = test.NewCluster(1)
h.API.Cluster.SetState(pilosa.ClusterStateResizing)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"info":"no resize job currently running"}`+"\n" {
bod, err := ioutil.ReadAll(w.Body)
t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err)
} else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
@ -186,8 +188,8 @@ func TestHandler_MaxSlices(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8)
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != http.StatusOK {
@ -227,8 +229,8 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
if w.Code != http.StatusOK {
@ -244,8 +246,8 @@ func TestHandler_Query_Args_URL(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
t.Fatalf("unexpected index: %s", index)
@ -272,8 +274,8 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
t.Fatalf("unexpected index: %s", index)
@ -312,8 +314,8 @@ func TestHandler_Query_Args_Err(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusBadRequest {
@ -339,8 +341,8 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
@ -360,8 +362,8 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
@ -390,8 +392,8 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
@ -423,8 +425,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
@ -446,8 +448,8 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
@ -494,8 +496,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
@ -555,8 +557,8 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{ID: 1, Count: 2},
@ -579,8 +581,8 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{ID: 1, Count: 2},
@ -612,15 +614,15 @@ func TestHandler_Query_Err_JSON(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`)))
if w.Code != http.StatusInternalServerError {
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
@ -633,8 +635,8 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
@ -643,7 +645,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) {
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusInternalServerError {
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
@ -661,8 +663,8 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil))
if w.Code != http.StatusMethodNotAllowed {
@ -676,8 +678,8 @@ func TestHandler_Query_ErrParse(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != http.StatusBadRequest {
@ -693,7 +695,7 @@ func TestHandler_Index_Delete(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Create index.
@ -733,8 +735,8 @@ func TestHandler_DeleteFrame(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader("")))
if w.Code != http.StatusOK {
@ -746,57 +748,13 @@ func TestHandler_DeleteFrame(t *testing.T) {
}
}
// Ensure handler can set the Index time quantum.
func TestHandler_SetIndexTimeQuantum(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if q := hldr.Index("i0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected time quantum: %s", q)
}
}
// Ensure handler can set the frame time quantum.
func TestHandler_SetFrameTimeQuantum(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Create frame.
if _, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if q := hldr.Index("i0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected time quantum: %s", q)
}
}
// Ensure the handler can return data in differing blocks for an index.
func TestHandler_Index_AttrStore_Diff(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set attributes on the index.
@ -845,7 +803,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set attributes on the index.
@ -895,12 +853,12 @@ func TestHandler_Frame_AddField(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
}
@ -925,7 +883,7 @@ func TestHandler_Frame_AddField(t *testing.T) {
t.Run("ErrInvalidFieldType", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil {
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
@ -947,7 +905,7 @@ func TestHandler_Frame_AddField(t *testing.T) {
t.Run("ErrInvalidFieldRange", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil {
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
@ -970,8 +928,7 @@ func TestHandler_Frame_AddField(t *testing.T) {
t.Run("ErrFieldAlreadyExists", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}},
Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}},
}); err != nil {
t.Fatal(err)
}
@ -999,12 +956,12 @@ func TestHandler_Frame_DeleteField(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil {
@ -1032,7 +989,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) {
t.Run("ErrFieldNotFound", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil {
@ -1064,12 +1021,12 @@ func TestHandler_Frame_GetFields(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 1, Max: 100}); err != nil {
@ -1103,7 +1060,7 @@ func TestHandler_Frame_GetFields(t *testing.T) {
t.Run("ErrFrameFieldNotAllowed", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
_, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{RangeEnabled: false})
_, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{})
resp, err := http.Get(s.URL + "/index/i/frame/f1/fields")
if err != nil {
@ -1111,12 +1068,12 @@ func TestHandler_Frame_GetFields(t *testing.T) {
}
if err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusBadRequest {
} else if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
} else if body, err := ioutil.ReadAll(resp.Body); err != nil {
t.Fatal(err)
} else if strings.TrimSpace(string(body)) != `frame fields not allowed` {
t.Fatalf("unexpected body: %q", body)
} else if strings.TrimSpace(string(body)) == `frame fields not allowed` {
t.Fatalf("shouldn't get frame fields not allowed error: %q", body)
}
})
@ -1132,7 +1089,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set bits in the index.
@ -1181,8 +1138,8 @@ func TestHandler_Version(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/version", nil)
@ -1204,9 +1161,9 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(3)
h.Cluster.ReplicaN = 2
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(3)
h.API.Cluster.ReplicaN = 2
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil)
@ -1225,6 +1182,13 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
// index is required
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
// Ensure the handler can return expvars without panicking.
@ -1233,8 +1197,8 @@ func TestHandler_Expvars(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/debug/vars", nil)
h.ServeHTTP(w, r)
@ -1279,8 +1243,8 @@ func TestHandler_CreateInputDefinition(t *testing.T) {
]
}`)
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody)))
if w.Code != http.StatusOK {
@ -1314,8 +1278,8 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) {
defer hldr.Close()
hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
//Ensure throwing error if there's duplicated primaryKey field
invalidPrimaryKey := []byte(`
@ -1419,8 +1383,8 @@ func TestHandler_DeleteInputDefinition(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
// Test index not found.
w := httptest.NewRecorder()
@ -1467,8 +1431,8 @@ func TestHandler_GetInputDefinition(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
@ -1634,8 +1598,8 @@ func TestHandler_CreateInput(t *testing.T) {
"null_value": null
}]`)
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
// Return error if index does not exist.
w := httptest.NewRecorder()
@ -1749,8 +1713,8 @@ func TestInput_JSON(t *testing.T) {
err: "set-timestamp value must be in time format: YYYY-MM-DD, has: 12345"},
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
for _, req := range tests {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(req.json))))
@ -1810,8 +1774,8 @@ func TestHandler_DeleteView(t *testing.T) {
hldr.Index("i0").Frame("f0").SetTimeQuantum("YMD")
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f0/view/standard_2017", strings.NewReader("")))
if w.Code != http.StatusOK {
@ -1836,8 +1800,8 @@ func TestHandler_RecalculateCaches(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil))
@ -1852,8 +1816,8 @@ func TestHandler_WebUI(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.FileSystem = &statik.FileSystem{}
w := httptest.NewRecorder()

96
holder.go
View file

@ -16,7 +16,6 @@ package pilosa
import (
"context"
"errors"
"fmt"
"io/ioutil"
"net/http"
@ -30,6 +29,7 @@ import (
"time"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
)
@ -47,7 +47,6 @@ type Holder struct {
// Indexes by name.
indexes map[string]*Index
hasData bool
// opened channel is closed once Open() completes.
opened chan struct{}
@ -91,36 +90,6 @@ func NewHolder() *Holder {
}
}
// Peek reads the root data directory for the holder
// without actually loading any data into memory.
// HasData is returned, and h.hasData is set.
func (h *Holder) Peek() bool {
h.Logger.Printf("peek at holder path: %s", h.Path)
h.hasData = false
// Open path to read all index directories.
f, err := os.Open(h.Path)
if err != nil {
return false
}
defer f.Close()
fis, err := f.Readdir(0)
if err != nil {
return false
}
for _, fi := range fis {
if !fi.IsDir() {
continue
}
h.hasData = true
break
}
return h.hasData
}
// Open initializes the root data directory for the holder.
func (h *Holder) Open() error {
h.setFileLimit()
@ -198,10 +167,37 @@ func (h *Holder) Close() error {
// HasData returns true if Holder contains at least one index.
// This is used to determine if the rebalancing of data is necessary
// when a node joins the cluster.
func (h *Holder) HasData() bool {
h.mu.RLock()
defer h.mu.RUnlock()
return h.hasData || len(h.indexes) > 0
func (h *Holder) HasData() (bool, error) {
h.mu.Lock()
defer h.mu.Unlock()
if len(h.indexes) > 0 {
return true, nil
}
// Open path to read all index directories.
if _, err := os.Stat(h.Path); os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, errors.Wrap(err, "statting data dir")
}
f, err := os.Open(h.Path)
if err != nil {
return false, errors.Wrap(err, "opening data dir")
}
defer f.Close()
fis, err := f.Readdir(0)
if err != nil {
return false, errors.Wrap(err, "reading data dir")
}
for _, fi := range fis {
if !fi.IsDir() {
continue
}
return true, nil
}
return false, nil
}
// MaxSlices returns MaxSlice map for all indexes.
@ -359,7 +355,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
}
// Update options.
index.SetTimeQuantum(opt.TimeQuantum)
h.indexes[index.Name()] = index
@ -529,9 +524,8 @@ func (h *Holder) setFileLimit() {
}
func (h *Holder) loadNodeID() (string, error) {
idPath := path.Join(h.Path, "ID")
idPath := path.Join(h.Path, ".id")
nodeID := ""
h.Logger.Printf("load NodeID: %s", idPath)
if err := os.MkdirAll(h.Path, 0777); err != nil {
return "", err
@ -553,6 +547,28 @@ func (h *Holder) loadNodeID() (string, error) {
return nodeID, nil
}
// Log startup time and version to $DATA_DIR/.startup.log
func (h *Holder) logStartup() error {
time, err := time.Now().MarshalText()
if err != nil {
return errors.Wrap(err, "creating timestamp")
}
logLine := fmt.Sprintf("%s\t%s\n", time, Version)
f, err := os.OpenFile(h.Path+"/.startup.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return errors.Wrap(err, "opening startup log")
}
defer f.Close()
if _, err = f.WriteString(logLine); err != nil {
return errors.Wrap(err, "writing startup log")
}
return nil
}
// HolderSyncer is an active anti-entropy tool that compares the local holder
// with a remote holder based on block checksums and resolves differences.
type HolderSyncer struct {
@ -614,7 +630,7 @@ func (s *HolderSyncer) SyncHolder() error {
for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ {
// Ignore slices that this host doesn't own.
if !s.Cluster.OwnsFragment(s.Node.ID, di.Name, slice) {
if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) {
continue
}

52
holder_test.go
View file

@ -25,6 +25,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
@ -73,22 +74,6 @@ func TestHolder_Open(t *testing.T) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrIndexMetaCorrupt", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if _, err := h.CreateIndex("test", pilosa.IndexOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".meta"), 2); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "unexpected EOF") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
@ -298,26 +283,24 @@ func TestHolder_HasData(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if h.HasData() {
t.Fatal("expected HasData to return false")
if ok, err := h.HasData(); ok || err != nil {
t.Fatal("expected HasData to return false, no err, but", ok, err)
}
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
if !h.HasData() {
t.Fatal("expected HasData to return true")
if ok, err := h.HasData(); !ok || err != nil {
t.Fatal("expected HasData to return true, but ", ok, err)
}
})
t.Run("Peek", func(t *testing.T) {
h := test.NewHolder()
if hasData := h.Peek(); hasData != false {
t.Fatal("expected Peek to return false")
} else if h.HasData() {
t.Fatal("expected HasData to return false")
if ok, err := h.HasData(); ok || err != nil {
t.Fatal("expected HasData to return false, no err, but", ok, err)
}
// Create an index directory to indicate data exists.
@ -325,24 +308,19 @@ func TestHolder_HasData(t *testing.T) {
t.Fatal(err)
}
if hasData := h.Peek(); hasData != true {
t.Fatal("expected Peek to return true")
} else if !h.HasData() {
t.Fatal("expected HasData to return true")
if ok, err := h.HasData(); !ok || err != nil {
t.Fatal("expected HasData to return true, no err, but", ok, err)
}
})
t.Run("Peek at missing directory", func(t *testing.T) {
h := test.NewHolder()
// Ensure that hasData is false when trying to peek into
// a directory that doesn't exist.
// Ensure that hasData is false when dir doesn't exist.
h.Path = "bad-path"
if hasData := h.Peek(); hasData != false {
t.Fatal("expected Peek to return false")
} else if h.HasData() {
t.Fatal("expected HasData to return false")
if ok, err := h.HasData(); ok || err != nil {
t.Fatal("expected HasData to return false, no err, but", ok, err)
}
})
}
@ -383,7 +361,7 @@ func TestHolder_DeleteIndex(t *testing.T) {
// Ensure holder can sync with a remote holder.
func TestHolderSyncer_SyncHolder(t *testing.T) {
cluster := test.NewCluster(2)
client := pilosa.GetHTTPClient(nil)
client := server.GetHTTPClient(nil)
// Create a local holder.
hldr0 := test.MustOpenHolder()
defer hldr0.Close()
@ -393,7 +371,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
defer hldr1.Close()
s := test.NewServer()
defer s.Close()
s.Handler.Holder = hldr1.Holder
s.Handler.API.Holder = hldr1.Holder
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(client)
e.Holder = hldr1.Holder
@ -467,7 +445,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
Holder: hldr0.Holder,
Node: cluster.Nodes[0],
Cluster: cluster,
RemoteClient: pilosa.GetHTTPClient(nil),
RemoteClient: server.GetHTTPClient(nil),
Stats: pilosa.NopStatsClient,
}

114
index.go
View file

@ -39,10 +39,6 @@ type Index struct {
path string
name string
// Default time quantum for all frames in index.
// This can be overridden by individual frames.
timeQuantum TimeQuantum
// Frames by name.
frames map[string]*Frame
@ -106,9 +102,7 @@ func (i *Index) Options() IndexOptions {
}
func (i *Index) options() IndexOptions {
return IndexOptions{
TimeQuantum: i.timeQuantum,
}
return IndexOptions{}
}
// Open opens and initializes the index.
@ -175,7 +169,6 @@ func (i *Index) loadMeta() error {
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta"))
if os.IsNotExist(err) {
i.timeQuantum = ""
return nil
} else if err != nil {
return err
@ -186,17 +179,18 @@ func (i *Index) loadMeta() error {
}
// Copy metadata fields.
i.timeQuantum = TimeQuantum(pb.TimeQuantum)
return nil
}
// NOTE: Until we introduce new attributes to store in the index .meta file,
// we don't need to actually write the file. The code related to index.options
// and the index meta file are left in place for future use.
/*
// saveMeta writes meta data for the index.
func (i *Index) saveMeta() error {
// Marshal metadata.
buf, err := proto.Marshal(&internal.IndexMeta{
TimeQuantum: string(i.timeQuantum),
})
buf, err := proto.Marshal(&internal.IndexMeta{})
if err != nil {
return err
}
@ -208,6 +202,7 @@ func (i *Index) saveMeta() error {
return nil
}
*/
// Close closes the index and its frames.
func (i *Index) Close() error {
@ -278,34 +273,6 @@ func (i *Index) SetRemoteMaxInverseSlice(v uint64) {
i.remoteMaxInverseSlice = v
}
// TimeQuantum returns the default time quantum for the index.
func (i *Index) TimeQuantum() TimeQuantum {
i.mu.RLock()
defer i.mu.RUnlock()
return i.timeQuantum
}
// SetTimeQuantum sets the default time quantum for the index.
func (i *Index) SetTimeQuantum(q TimeQuantum) error {
i.mu.Lock()
defer i.mu.Unlock()
// Validate input.
if !q.Valid() {
return ErrInvalidTimeQuantum
}
// Update value on index.
i.timeQuantum = q
// Perist meta data to disk.
if err := i.saveMeta(); err != nil {
return err
}
return nil
}
// FramePath returns the path to a frame in the index.
func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) }
@ -404,13 +371,7 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
// Validate mutually exclusive options if ranges are enabled.
if opt.RangeEnabled {
if opt.InverseEnabled {
return nil, ErrInverseRangeNotAllowed
}
} else {
if len(opt.Fields) > 0 {
return nil, ErrFrameFieldsNotAllowed
}
i.Logger.Printf("RangeEnabled is deprecated - no need to set RangeEnabled to true when creating a frame")
}
// Validate fields.
@ -431,12 +392,8 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
return nil, err
}
// Default the time quantum to what is set on the Index.
timeQuantum := i.timeQuantum
if opt.TimeQuantum != "" {
timeQuantum = opt.TimeQuantum
}
if err := f.SetTimeQuantum(timeQuantum); err != nil {
// Set the time quantum.
if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil {
f.Close()
return nil, err
}
@ -452,9 +409,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
}
f.inverseEnabled = opt.InverseEnabled
f.rangeEnabled = opt.RangeEnabled
f.rangeEnabled = opt.RangeEnabled
// Set fields.
f.fields = opt.Fields
@ -527,46 +481,6 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p indexInfoSlice) Len() int { return len(p) }
func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// MergeSchemas combines indexes and frames from a and b into one schema.
func MergeSchemas(a, b []*IndexInfo) []*IndexInfo {
// Generate a map from both schemas.
m := make(map[string]map[string]map[string]struct{})
for _, idxs := range [][]*IndexInfo{a, b} {
for _, idx := range idxs {
if m[idx.Name] == nil {
m[idx.Name] = make(map[string]map[string]struct{})
}
for _, frame := range idx.Frames {
if m[idx.Name][frame.Name] == nil {
m[idx.Name][frame.Name] = make(map[string]struct{})
}
for _, view := range frame.Views {
m[idx.Name][frame.Name][view.Name] = struct{}{}
}
}
}
}
// Generate new schema from map.
idxs := make([]*IndexInfo, 0, len(m))
for idx, frames := range m {
di := &IndexInfo{Name: idx}
for frame, views := range frames {
fi := &FrameInfo{Name: frame}
for view := range views {
fi.Views = append(fi.Views, &ViewInfo{Name: view})
}
sort.Sort(viewInfoSlice(fi.Views))
di.Frames = append(di.Frames, fi)
}
sort.Sort(frameInfoSlice(di.Frames))
idxs = append(idxs, di)
}
sort.Sort(indexInfoSlice(idxs))
return idxs
}
// EncodeIndexes converts a into its internal representation.
func EncodeIndexes(a []*Index) []*internal.Index {
other := make([]*internal.Index, len(a))
@ -586,15 +500,11 @@ func encodeIndex(d *Index) *internal.Index {
}
// IndexOptions represents options to set when initializing an index.
type IndexOptions struct {
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
}
type IndexOptions struct{}
// Encode converts i into its internal representation.
func (i *IndexOptions) Encode() *internal.IndexMeta {
return &internal.IndexMeta{
TimeQuantum: string(i.TimeQuantum),
}
return &internal.IndexMeta{}
}
// hasTime returns true if a contains a non-nil time.

107
index_test.go
View file

@ -58,11 +58,6 @@ func TestIndex_CreateFrame(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Set index time quantum.
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
t.Fatal(err)
}
// Create frame with explicit quantum.
f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")})
if err != nil {
@ -71,24 +66,6 @@ func TestIndex_CreateFrame(t *testing.T) {
t.Fatalf("unexpected frame time quantum: %s", q)
}
})
t.Run("Inherited", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Set index time quantum.
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
t.Fatal(err)
}
// Create frame.
f, err := index.CreateFrame("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") {
t.Fatalf("unexpected frame time quantum: %s", q)
}
})
})
// Ensure frame can include range columns.
@ -99,7 +76,7 @@ func TestIndex_CreateFrame(t *testing.T) {
// Create frame with schema and verify it exists.
if f, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
RangeEnabled: false,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
@ -124,16 +101,47 @@ func TestIndex_CreateFrame(t *testing.T) {
}
})
t.Run("ErrInverseRangeNotAllowed", func(t *testing.T) {
t.Run("ErrInverseRangeAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
InverseEnabled: true,
frame, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
}); err != pilosa.ErrInverseRangeNotAllowed {
InverseEnabled: true,
Fields: []*pilosa.Field{
&pilosa.Field{
Name: "myfield",
Type: pilosa.FieldTypeInt,
Min: -20,
Max: 100,
},
},
})
if err != nil {
t.Fatal(err)
}
ch, err := frame.SetBit(pilosa.ViewStandard, 1, 2, nil)
if !ch || err != nil {
t.Fatal(ch, err)
}
ch, err = frame.SetBit(pilosa.ViewInverse, 1, 2, nil)
if !ch || err != nil {
t.Fatal(ch, err)
}
ch, err = frame.SetFieldValue(1, "myfield", 87)
if !ch || err != nil {
t.Fatal(ch, err)
}
views := frame.Views()
if len(views) != 3 {
var names string
for _, v := range views {
names = names + v.Name() + " "
}
t.Fatalf("Unexpected views: %s", names)
}
})
t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
@ -141,8 +149,7 @@ func TestIndex_CreateFrame(t *testing.T) {
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
CacheType: pilosa.CacheTypeRanked,
CacheType: pilosa.CacheTypeRanked,
}); err != nil {
t.Fatal(err)
}
@ -152,15 +159,14 @@ func TestIndex_CreateFrame(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
CacheType: pilosa.CacheTypeNone,
CacheSize: uint32(5),
CacheType: pilosa.CacheTypeNone,
CacheSize: uint32(5),
}); err != nil {
t.Fatal(err)
}
})
t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) {
t.Run("ErrFrameFieldsAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
@ -168,7 +174,7 @@ func TestIndex_CreateFrame(t *testing.T) {
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt},
},
}); err != pilosa.ErrFrameFieldsNotAllowed {
}); err != nil {
t.Fatal(err)
}
})
@ -178,7 +184,6 @@ func TestIndex_CreateFrame(t *testing.T) {
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "", Type: pilosa.FieldTypeInt},
},
@ -192,7 +197,6 @@ func TestIndex_CreateFrame(t *testing.T) {
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: "bad_type"},
},
@ -206,7 +210,7 @@ func TestIndex_CreateFrame(t *testing.T) {
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
RangeEnabled: true, // make sure we can still create frames with RangeEnabled: true after deprecation
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50},
},
@ -240,26 +244,6 @@ func TestIndex_DeleteFrame(t *testing.T) {
}
}
// Ensure index can set the default time quantum.
func TestIndex_SetTimeQuantum(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Set & retrieve time quantum.
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
}
// Reload index and verify that it is persisted.
if err := index.Reopen(); err != nil {
t.Fatal(err)
} else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum (reopen): %s", q)
}
}
// Ensure index can delete a frame.
func TestIndex_InvalidName(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-index-")
@ -377,18 +361,13 @@ func TestIndex_InputBits(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Set index time quantum.
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
t.Fatal(err)
}
err := index.InputBits("f", bits)
if !strings.Contains(err.Error(), "Frame not found") {
t.Fatalf("Expected Frame not found error, actual error: %s", err)
}
// Create frame.
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YM")}); err != nil {
t.Fatal(err)
}

213
internal/private.pb.go
View file

@ -68,7 +68,6 @@ var _ = math.Inf
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type IndexMeta struct {
TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
}
func (m *IndexMeta) Reset() { *m = IndexMeta{} }
@ -76,13 +75,6 @@ func (m *IndexMeta) String() string { return proto.CompactTextString(
func (*IndexMeta) ProtoMessage() {}
func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} }
func (m *IndexMeta) GetTimeQuantum() string {
if m != nil {
return m.TimeQuantum
}
return ""
}
type FrameMeta struct {
InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"`
CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
@ -1232,12 +1224,6 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.TimeQuantum) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum)))
i += copy(dAtA[i:], m.TimeQuantum)
}
return i, nil
}
@ -2746,10 +2732,6 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int {
func (m *IndexMeta) Size() (n int) {
var l int
_ = l
l = len(m.TimeQuantum)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
return n
}
@ -3439,35 +3421,6 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error {
return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.TimeQuantum = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -8573,87 +8526,87 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 1308 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45,
0x14, 0x67, 0xbd, 0xb6, 0x13, 0x3f, 0xd7, 0xa9, 0x33, 0x6d, 0x83, 0x5b, 0x45, 0xae, 0x19, 0x15,
0x1a, 0x2a, 0x35, 0x2a, 0xa9, 0x84, 0x68, 0xa1, 0x52, 0x69, 0xec, 0xaa, 0x0b, 0xa4, 0x2a, 0xe3,
0xb6, 0x48, 0x48, 0x20, 0x4d, 0xed, 0x21, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0x49, 0xdc, 0x03,
0x47, 0x84, 0x84, 0xb8, 0x23, 0xae, 0x7c, 0x19, 0x8e, 0x7c, 0x02, 0x84, 0xc2, 0x87, 0xe0, 0x08,
0x9a, 0x37, 0x33, 0xbb, 0xeb, 0x7f, 0x49, 0x13, 0xb8, 0xed, 0xfb, 0xff, 0x9b, 0xf7, 0x6f, 0x66,
0xa1, 0x36, 0x8c, 0xfd, 0x7d, 0x2e, 0xc5, 0xe6, 0x30, 0x8e, 0x64, 0x44, 0x96, 0xfd, 0x50, 0x8a,
0x38, 0xe4, 0x01, 0xbd, 0x09, 0x15, 0x2f, 0xec, 0x8b, 0xc3, 0x1d, 0x21, 0x39, 0x69, 0x41, 0xf5,
0xa9, 0x3f, 0x10, 0x9f, 0x8f, 0x78, 0x28, 0x47, 0x83, 0x46, 0xa1, 0xe5, 0x6c, 0x54, 0x58, 0x9e,
0x45, 0xff, 0x70, 0xa0, 0xf2, 0x30, 0xe6, 0x03, 0x81, 0xfa, 0xef, 0xc0, 0x8a, 0x17, 0xee, 0x8b,
0x38, 0x11, 0x9d, 0x90, 0xbf, 0x08, 0x44, 0x1f, 0x4d, 0x96, 0xd9, 0x14, 0x97, 0xac, 0x43, 0x65,
0x9b, 0xf7, 0x5e, 0x8a, 0xa7, 0xe3, 0xa1, 0x68, 0xb8, 0xe8, 0x35, 0x63, 0xa4, 0xd2, 0xae, 0xff,
0x4a, 0x34, 0x8a, 0x2d, 0x67, 0xa3, 0xc6, 0x32, 0xc6, 0x34, 0xa6, 0xd2, 0x0c, 0x26, 0x42, 0xe1,
0x1c, 0xe3, 0xe1, 0x6e, 0x8a, 0xa1, 0x8c, 0x18, 0x26, 0x78, 0xe4, 0x3a, 0x94, 0x1f, 0xfa, 0x22,
0xe8, 0x27, 0x8d, 0xa5, 0x96, 0xbb, 0x51, 0xdd, 0x3a, 0xbf, 0x69, 0x33, 0xb0, 0x89, 0x7c, 0x66,
0xc4, 0x94, 0xc2, 0x8a, 0x37, 0x18, 0x46, 0xb1, 0x64, 0x22, 0x19, 0x46, 0x61, 0x22, 0x48, 0x1d,
0xdc, 0x4e, 0x1c, 0x37, 0x1c, 0x0c, 0xac, 0x3e, 0xe9, 0x77, 0x50, 0x7f, 0x10, 0x44, 0xbd, 0xbd,
0x36, 0x97, 0x9c, 0x89, 0x6f, 0x47, 0x22, 0x91, 0xe4, 0x22, 0x94, 0x30, 0x8f, 0x46, 0x4f, 0x13,
0x8a, 0x8b, 0xd9, 0x32, 0xa9, 0xd4, 0x84, 0xe2, 0xa2, 0x3d, 0xa6, 0xa2, 0xc8, 0x34, 0xa1, 0xb8,
0xdd, 0xc0, 0xef, 0xe9, 0x14, 0x14, 0x99, 0x26, 0x08, 0x81, 0xe2, 0x73, 0x5f, 0x1c, 0x98, 0x73,
0xe3, 0x37, 0xf5, 0x60, 0x35, 0x17, 0xdf, 0xc0, 0x5c, 0x83, 0x32, 0x8b, 0x0e, 0xbc, 0x76, 0xd2,
0x70, 0x5a, 0xee, 0x46, 0x91, 0x19, 0x0a, 0xb3, 0x1b, 0x05, 0xa3, 0x41, 0xa8, 0x44, 0x05, 0x14,
0x65, 0x0c, 0x7a, 0x19, 0x4a, 0x98, 0x6a, 0x75, 0xca, 0xcc, 0x56, 0x7d, 0xd2, 0x7f, 0x1c, 0xa8,
0xec, 0xf0, 0x43, 0x84, 0x91, 0x90, 0x7b, 0xb0, 0xdc, 0x95, 0x3c, 0xec, 0xf3, 0xb8, 0x8f, 0x4a,
0xd5, 0xad, 0xb7, 0xb2, 0x14, 0xa6, 0x6a, 0x9b, 0x56, 0xa7, 0x13, 0xca, 0x78, 0xcc, 0x52, 0x13,
0x72, 0x17, 0x96, 0x4c, 0x4f, 0x20, 0x86, 0xea, 0x56, 0x6b, 0x9e, 0x75, 0xda, 0x36, 0xca, 0xd8,
0x1a, 0x5c, 0xf9, 0x10, 0x6a, 0x13, 0x6e, 0x15, 0xd6, 0x3d, 0x31, 0xb6, 0x15, 0xd9, 0x13, 0x63,
0x95, 0xbb, 0x7d, 0x1e, 0x8c, 0x74, 0x9e, 0x8b, 0x4c, 0x13, 0x77, 0x0b, 0x1f, 0x38, 0x57, 0xee,
0xc2, 0xb9, 0xbc, 0xd7, 0xd3, 0xd8, 0xd2, 0xaf, 0x81, 0x6c, 0xc7, 0x82, 0x4b, 0x81, 0xf0, 0x76,
0x44, 0x92, 0xf0, 0x5d, 0xb1, 0xb8, 0xd2, 0xba, 0x7a, 0x85, 0x7c, 0xf5, 0xd6, 0xa1, 0xe2, 0x25,
0xf6, 0xe0, 0x2e, 0xf6, 0x65, 0xc6, 0xa0, 0x37, 0x80, 0xb4, 0x45, 0x20, 0xa4, 0x30, 0x13, 0x78,
0x8c, 0x7f, 0xda, 0xb5, 0x58, 0x4e, 0xd6, 0x25, 0xd7, 0xa1, 0xa8, 0xc6, 0x13, 0xa1, 0x54, 0xb7,
0x2e, 0x64, 0x99, 0x4e, 0x27, 0x9d, 0xa1, 0x02, 0xf5, 0xad, 0x53, 0x33, 0xd2, 0x27, 0x1c, 0x70,
0x4e, 0x2b, 0xdb, 0x50, 0xee, 0x74, 0xa8, 0x74, 0x49, 0x98, 0x50, 0xf7, 0xed, 0x59, 0xcf, 0x1a,
0x8a, 0xee, 0xa6, 0x60, 0xd5, 0xa4, 0x9e, 0x05, 0xec, 0xdb, 0x50, 0x42, 0x5b, 0x83, 0x76, 0x66,
0x07, 0x68, 0x29, 0x7d, 0x9e, 0x42, 0x3d, 0x6b, 0xa0, 0x8b, 0xf9, 0x40, 0x15, 0xeb, 0xf7, 0x4b,
0xa3, 0xab, 0x66, 0xfa, 0xb1, 0xb2, 0xd1, 0x9e, 0xf0, 0x7b, 0x71, 0xcd, 0xa6, 0x12, 0xa9, 0x7c,
0xab, 0x25, 0x90, 0x34, 0xdc, 0x96, 0xab, 0x7c, 0x23, 0x41, 0x6f, 0x43, 0xb9, 0xdb, 0x7b, 0x29,
0x06, 0x9c, 0xbc, 0xab, 0x26, 0xad, 0x2f, 0x0e, 0x45, 0x62, 0xe6, 0xf4, 0xfc, 0x54, 0xfd, 0x99,
0x95, 0xd3, 0x1f, 0x1d, 0x73, 0xa6, 0x05, 0x88, 0xca, 0x18, 0x3b, 0x69, 0x14, 0x67, 0x56, 0xa6,
0xe2, 0x33, 0x23, 0x26, 0x1d, 0xa8, 0x7b, 0xe1, 0x70, 0x24, 0xdb, 0xe2, 0x1b, 0x3f, 0xf4, 0xa5,
0x1f, 0x85, 0x49, 0xa3, 0x8c, 0x26, 0x97, 0xf3, 0xa1, 0x27, 0x34, 0xd8, 0x8c, 0x09, 0xfd, 0xde,
0x81, 0xf3, 0x53, 0xcc, 0x13, 0x70, 0x15, 0x8e, 0xc7, 0xf5, 0x7e, 0xba, 0xf3, 0x5d, 0x54, 0x6c,
0x2e, 0x44, 0x33, 0x79, 0x05, 0xfc, 0xea, 0xc0, 0xc5, 0x79, 0x0a, 0x73, 0xd1, 0x34, 0x01, 0x9e,
0xc4, 0xfe, 0x80, 0xc7, 0xe3, 0x4f, 0xc5, 0xd8, 0x5c, 0x7f, 0x39, 0x0e, 0xf9, 0x02, 0xd6, 0xa6,
0x7c, 0x7d, 0xdc, 0xd3, 0x29, 0xd2, 0xa0, 0xae, 0x2e, 0x04, 0xa5, 0xf5, 0xd8, 0x02, 0x73, 0xfa,
0xb7, 0x03, 0x97, 0xe6, 0x8a, 0xb2, 0x9e, 0x74, 0xf2, 0x3d, 0x79, 0x03, 0xea, 0xcf, 0xd5, 0x66,
0x6b, 0x8b, 0x44, 0xfa, 0x21, 0x57, 0x9a, 0xa6, 0x69, 0x67, 0xf8, 0xc4, 0x83, 0x65, 0xe4, 0xed,
0xf0, 0xa1, 0x81, 0x79, 0xf3, 0x04, 0x98, 0x9b, 0x56, 0xdf, 0x2c, 0x7e, 0x4b, 0x2a, 0x30, 0x78,
0x11, 0xd9, 0x5b, 0x0d, 0x09, 0xb5, 0xd2, 0x27, 0x0c, 0x4e, 0xb5, 0x96, 0x23, 0x58, 0xb7, 0xab,
0x70, 0x02, 0xc9, 0xf1, 0x93, 0x7a, 0x07, 0x20, 0x53, 0x35, 0x1b, 0xe0, 0x98, 0xfe, 0xcc, 0x29,
0xd3, 0x47, 0xb0, 0x6e, 0xf7, 0xf4, 0x29, 0x02, 0xda, 0x6e, 0x29, 0x64, 0xdd, 0x42, 0x3b, 0xe0,
0x3e, 0x63, 0x9e, 0xba, 0xab, 0x71, 0x5a, 0x6d, 0x89, 0x0c, 0xa5, 0x4c, 0x1e, 0x45, 0x89, 0xb4,
0x26, 0xea, 0x5b, 0xf1, 0x9e, 0x44, 0xb1, 0x44, 0xc4, 0x35, 0x86, 0xdf, 0xf4, 0x2b, 0x28, 0x3e,
0x8e, 0xfa, 0x82, 0xac, 0x40, 0xc1, 0x6b, 0x1b, 0x1f, 0x05, 0xaf, 0x4d, 0xae, 0xa2, 0x7b, 0xb3,
0x43, 0x6a, 0xd9, 0xe1, 0x9e, 0x31, 0x8f, 0x61, 0xe0, 0x6b, 0x50, 0xf3, 0x92, 0xed, 0x28, 0x8a,
0xfb, 0xaa, 0xd4, 0x51, 0x6c, 0xee, 0xa4, 0x49, 0x26, 0xbd, 0x0f, 0x75, 0xe5, 0xbe, 0x2b, 0xb9,
0x4c, 0x37, 0xf5, 0x1a, 0x94, 0x15, 0x2f, 0x0d, 0x67, 0x28, 0xbc, 0xf7, 0x94, 0x9e, 0x5d, 0x80,
0x48, 0xd0, 0xcf, 0xb4, 0x87, 0xce, 0xbe, 0x08, 0x65, 0x2e, 0x4b, 0x48, 0xa3, 0x83, 0x1a, 0xd3,
0x04, 0xa1, 0xfa, 0x28, 0x06, 0xf3, 0x4a, 0x86, 0x59, 0x71, 0x19, 0xca, 0xe8, 0x4f, 0x0e, 0x80,
0x05, 0x34, 0x4a, 0x52, 0x13, 0x67, 0xb1, 0x09, 0x79, 0x2f, 0xf7, 0x76, 0x99, 0xdd, 0xa9, 0xa9,
0x88, 0xe5, 0x5e, 0x38, 0x1b, 0x76, 0x85, 0x9a, 0xe6, 0xa8, 0x67, 0xfa, 0x9a, 0x6f, 0xca, 0xa4,
0xae, 0xcd, 0xda, 0x76, 0x30, 0x4a, 0xa4, 0x88, 0x0d, 0x22, 0xf5, 0xc6, 0xd2, 0x8c, 0x34, 0x3f,
0x19, 0x63, 0x7e, 0x8a, 0xc8, 0x35, 0x28, 0x29, 0xa4, 0x76, 0x0f, 0x4c, 0x1f, 0x43, 0x0b, 0x69,
0xd7, 0xdc, 0x24, 0x73, 0x77, 0x0f, 0x81, 0x22, 0xbe, 0xa8, 0x4d, 0xbb, 0xe0, 0x63, 0xba, 0x0e,
0xee, 0x8e, 0xaf, 0xfb, 0xdb, 0x65, 0xea, 0x13, 0x39, 0xfc, 0x10, 0xe7, 0x4f, 0x71, 0xb8, 0x7a,
0x4b, 0xac, 0xea, 0x01, 0x52, 0x77, 0xc7, 0x59, 0xee, 0x37, 0xfb, 0x28, 0x75, 0x73, 0x8f, 0xd2,
0x2e, 0xac, 0xea, 0x21, 0xf9, 0x3f, 0x9d, 0xfe, 0x52, 0x80, 0x55, 0x26, 0x12, 0xff, 0x95, 0xf0,
0xc2, 0x44, 0xc6, 0xa3, 0x74, 0xc1, 0x7d, 0x12, 0xbd, 0x30, 0xa9, 0x76, 0x99, 0x26, 0x5e, 0xa7,
0x93, 0xc8, 0x2d, 0xa8, 0x4e, 0x77, 0xff, 0xac, 0x6a, 0x5e, 0x85, 0xdc, 0x82, 0xa5, 0x6e, 0x34,
0x8a, 0x7b, 0xe9, 0x35, 0xb8, 0x96, 0x69, 0x6b, 0x64, 0x5a, 0xcc, 0xac, 0x5a, 0xae, 0x8f, 0x4a,
0xc7, 0xf7, 0x11, 0xb9, 0x37, 0xd5, 0x47, 0xf8, 0xe7, 0x52, 0xdd, 0x7a, 0x33, 0x33, 0x98, 0x10,
0xb3, 0x49, 0x6d, 0xfa, 0x83, 0x03, 0xe7, 0xf2, 0x10, 0x5e, 0x6b, 0x30, 0xd2, 0x8a, 0x14, 0xe6,
0x56, 0xc4, 0x9d, 0x57, 0x91, 0x62, 0x56, 0x91, 0xec, 0x9d, 0x5b, 0xca, 0xbd, 0x73, 0xe9, 0x1e,
0x5c, 0x9e, 0x29, 0xd3, 0x76, 0x34, 0x18, 0xaa, 0x7e, 0xf8, 0x0f, 0xe5, 0x52, 0x2b, 0x23, 0x8e,
0x4d, 0xa1, 0x2a, 0x4c, 0x13, 0xf4, 0x0e, 0x5c, 0xea, 0x0a, 0x99, 0x2b, 0x92, 0xed, 0xb6, 0x16,
0xb8, 0x8f, 0xc5, 0xc1, 0x82, 0xe3, 0x2b, 0x11, 0xfd, 0x08, 0x1a, 0xcf, 0x86, 0x7d, 0x2e, 0xc5,
0x99, 0xac, 0x1f, 0xc0, 0xf2, 0xd3, 0x68, 0x18, 0x05, 0xd1, 0xee, 0xf8, 0x84, 0x91, 0x6f, 0xc0,
0x92, 0xde, 0x8f, 0xfa, 0x91, 0x52, 0x61, 0x96, 0xa4, 0x17, 0x54, 0x43, 0xf7, 0x78, 0xd0, 0x1b,
0x05, 0x0a, 0x86, 0xfa, 0xf7, 0x4a, 0x1e, 0xd4, 0x7f, 0x3b, 0x6a, 0x3a, 0xbf, 0x1f, 0x35, 0x9d,
0x3f, 0x8f, 0x9a, 0xce, 0xcf, 0x7f, 0x35, 0xdf, 0x78, 0x51, 0xc6, 0xff, 0xf4, 0xdb, 0xff, 0x06,
0x00, 0x00, 0xff, 0xff, 0xda, 0x68, 0xc4, 0x54, 0xb8, 0x0f, 0x00, 0x00,
// 1307 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4d, 0x6f, 0x1b, 0xc5,
0x1b, 0xff, 0xaf, 0xd7, 0x76, 0xe2, 0xc7, 0x71, 0xe2, 0x4c, 0xd3, 0xfc, 0x37, 0x55, 0xe4, 0x9a,
0x51, 0xa1, 0xa6, 0x12, 0x51, 0x49, 0x25, 0x44, 0x03, 0x95, 0x4a, 0x62, 0x57, 0x5d, 0x20, 0x51,
0x19, 0x27, 0x41, 0x42, 0x02, 0x69, 0x62, 0x0f, 0xe9, 0x2a, 0xeb, 0x5d, 0xb3, 0x3b, 0x4e, 0xe2,
0x1e, 0x38, 0x22, 0x24, 0xc4, 0x1d, 0x71, 0xe5, 0xcb, 0x70, 0xe4, 0x13, 0x20, 0x14, 0x3e, 0x04,
0x47, 0xd0, 0xbc, 0xed, 0xae, 0xdf, 0x92, 0x26, 0x70, 0xdb, 0xe7, 0xfd, 0x37, 0xcf, 0xdb, 0xcc,
0x42, 0xa5, 0x1f, 0x79, 0xa7, 0x94, 0xb3, 0x8d, 0x7e, 0x14, 0xf2, 0x10, 0xcd, 0x7b, 0x01, 0x67,
0x51, 0x40, 0x7d, 0x5c, 0x86, 0x92, 0x1b, 0x74, 0xd9, 0xf9, 0x2e, 0xe3, 0x14, 0xff, 0x6e, 0x41,
0xe9, 0x59, 0x44, 0x7b, 0x4c, 0x50, 0xe8, 0x2d, 0x58, 0x74, 0x83, 0x53, 0x16, 0xc5, 0xac, 0x15,
0xd0, 0x23, 0x9f, 0x75, 0x9d, 0x5c, 0xdd, 0x6a, 0xcc, 0x93, 0x31, 0x2e, 0x5a, 0x87, 0xd2, 0x0e,
0xed, 0xbc, 0x64, 0xfb, 0xc3, 0x3e, 0x73, 0xec, 0xba, 0xd5, 0x28, 0x91, 0x94, 0x91, 0x48, 0xdb,
0xde, 0x2b, 0xe6, 0xe4, 0xeb, 0x56, 0xa3, 0x42, 0x52, 0x06, 0xaa, 0x43, 0x79, 0xdf, 0xeb, 0xb1,
0xcf, 0x06, 0x34, 0xe0, 0x83, 0x9e, 0x53, 0x90, 0xd6, 0x59, 0x16, 0xc2, 0xb0, 0x40, 0x68, 0x70,
0x9c, 0x60, 0x28, 0x4a, 0x0c, 0x23, 0x3c, 0x74, 0x1f, 0x8a, 0xcf, 0x3c, 0xe6, 0x77, 0x63, 0x67,
0xae, 0x6e, 0x37, 0xca, 0x9b, 0x4b, 0x1b, 0xe6, 0x7c, 0x1b, 0x92, 0x4f, 0xb4, 0x18, 0x63, 0x58,
0x74, 0x7b, 0xfd, 0x30, 0xe2, 0x84, 0xc5, 0xfd, 0x30, 0x88, 0x19, 0xaa, 0x82, 0xdd, 0x8a, 0x22,
0xc7, 0x92, 0x81, 0xc5, 0x27, 0xfe, 0x16, 0xaa, 0xdb, 0x7e, 0xd8, 0x39, 0x69, 0x52, 0x4e, 0x09,
0xfb, 0x66, 0xc0, 0x62, 0x8e, 0x56, 0xa0, 0x20, 0xb3, 0xa4, 0xf5, 0x14, 0x21, 0xb8, 0x32, 0x5b,
0x32, 0x2f, 0x25, 0xa2, 0x08, 0xc1, 0x95, 0xf6, 0x32, 0x15, 0x79, 0xa2, 0x08, 0xc1, 0x6d, 0xfb,
0x5e, 0x47, 0xa5, 0x20, 0x4f, 0x14, 0x81, 0x10, 0xe4, 0x0f, 0x3d, 0x76, 0xa6, 0xcf, 0x2d, 0xbf,
0xb1, 0x0b, 0xcb, 0x99, 0xf8, 0x1a, 0xe6, 0x2a, 0x14, 0x49, 0x78, 0xe6, 0x36, 0x63, 0xc7, 0xaa,
0xdb, 0x8d, 0x3c, 0xd1, 0x94, 0xcc, 0x6e, 0xe8, 0x0f, 0x7a, 0x81, 0x10, 0xe5, 0xa4, 0x28, 0x65,
0xe0, 0x35, 0x28, 0xc8, 0x54, 0x8b, 0x53, 0xa6, 0xb6, 0xe2, 0x13, 0xff, 0x6d, 0x41, 0x69, 0x97,
0x9e, 0x4b, 0x18, 0x31, 0x7a, 0x02, 0xf3, 0x6d, 0x4e, 0x83, 0x2e, 0x8d, 0xba, 0x52, 0xa9, 0xbc,
0xf9, 0x46, 0x9a, 0xc2, 0x44, 0x6d, 0xc3, 0xe8, 0xb4, 0x02, 0x1e, 0x0d, 0x49, 0x62, 0x82, 0xb6,
0x60, 0x4e, 0xf7, 0x84, 0xc4, 0x50, 0xde, 0xac, 0x4f, 0xb3, 0x4e, 0xda, 0x46, 0x18, 0x1b, 0x83,
0x3b, 0x1f, 0x40, 0x65, 0xc4, 0xad, 0xc0, 0x7a, 0xc2, 0x86, 0xa6, 0x22, 0x27, 0x6c, 0x28, 0x72,
0x77, 0x4a, 0xfd, 0x81, 0xca, 0x73, 0x9e, 0x28, 0x62, 0x2b, 0xf7, 0xbe, 0x75, 0x67, 0x0b, 0x16,
0xb2, 0x5e, 0xaf, 0x63, 0x8b, 0xbf, 0x02, 0xb4, 0x13, 0x31, 0xca, 0x99, 0x84, 0xb7, 0xcb, 0xe2,
0x98, 0x1e, 0xb3, 0xd9, 0x95, 0x56, 0xd5, 0xcb, 0x65, 0xab, 0xb7, 0x0e, 0x25, 0x37, 0x36, 0x07,
0xb7, 0x65, 0x5f, 0xa6, 0x0c, 0xfc, 0x00, 0x50, 0x93, 0xf9, 0x8c, 0x33, 0x3d, 0x5f, 0x97, 0xf8,
0xc7, 0x6d, 0x83, 0xe5, 0x6a, 0x5d, 0x74, 0x1f, 0xf2, 0x62, 0x3c, 0x25, 0x94, 0xf2, 0xe6, 0xad,
0x34, 0xd3, 0xc9, 0x1c, 0x13, 0xa9, 0x80, 0x3d, 0xe3, 0x54, 0x8f, 0xf4, 0x15, 0x07, 0x9c, 0xd2,
0xca, 0x26, 0x94, 0x3d, 0x1e, 0x2a, 0x59, 0x12, 0x3a, 0xd4, 0x53, 0x73, 0xd6, 0x9b, 0x86, 0xc2,
0xc7, 0x09, 0x58, 0x31, 0xa9, 0x37, 0x01, 0xfb, 0x26, 0x14, 0xa4, 0xad, 0x46, 0x3b, 0xb1, 0x03,
0x94, 0x14, 0x1f, 0x26, 0x50, 0x6f, 0x1a, 0x68, 0x25, 0x1b, 0xa8, 0x64, 0xfc, 0x7e, 0xa1, 0x75,
0xc5, 0x4c, 0xef, 0x09, 0x1b, 0xe5, 0x49, 0x7e, 0xcf, 0xae, 0xd9, 0x58, 0x22, 0x85, 0x6f, 0xb1,
0x04, 0x62, 0xc7, 0xae, 0xdb, 0xc2, 0xb7, 0x24, 0xf0, 0x23, 0x28, 0xb6, 0x3b, 0x2f, 0x59, 0x8f,
0xa2, 0xb7, 0xc5, 0xa4, 0x75, 0xd9, 0x39, 0x8b, 0xf5, 0x9c, 0x2e, 0x8d, 0xd5, 0x9f, 0x18, 0x39,
0xfe, 0xc1, 0xd2, 0x67, 0x9a, 0x81, 0xa8, 0x28, 0x63, 0xc7, 0x4e, 0x7e, 0x62, 0x65, 0x0a, 0x3e,
0xd1, 0x62, 0xd4, 0x82, 0xaa, 0x1b, 0xf4, 0x07, 0xbc, 0xc9, 0xbe, 0xf6, 0x02, 0x8f, 0x7b, 0x61,
0x10, 0x3b, 0x45, 0x69, 0xb2, 0x96, 0x0d, 0x3d, 0xa2, 0x41, 0x26, 0x4c, 0xf0, 0x77, 0x16, 0x2c,
0x8d, 0x31, 0xaf, 0xc0, 0x95, 0xbb, 0x1c, 0xd7, 0x7b, 0xc9, 0xce, 0xb7, 0xa5, 0x62, 0x6d, 0x26,
0x9a, 0xd1, 0x2b, 0xe0, 0x17, 0x0b, 0x56, 0xa6, 0x29, 0x4c, 0x45, 0x53, 0x03, 0x78, 0x11, 0x79,
0x3d, 0x1a, 0x0d, 0x3f, 0x61, 0x43, 0x7d, 0xfd, 0x65, 0x38, 0xe8, 0x73, 0x58, 0x1d, 0xf3, 0xf5,
0x51, 0x47, 0xa5, 0x48, 0x81, 0xba, 0x3b, 0x13, 0x94, 0xd2, 0x23, 0x33, 0xcc, 0xf1, 0x5f, 0x16,
0xdc, 0x9e, 0x2a, 0x4a, 0x7b, 0xd2, 0xca, 0xf6, 0xe4, 0x03, 0xa8, 0x1e, 0x8a, 0xcd, 0xd6, 0x64,
0x31, 0xf7, 0x02, 0x2a, 0x34, 0x75, 0xd3, 0x4e, 0xf0, 0x91, 0x0b, 0xf3, 0x92, 0xb7, 0x4b, 0xfb,
0x1a, 0xe6, 0x3b, 0x57, 0xc0, 0xdc, 0x30, 0xfa, 0x7a, 0xf1, 0x1b, 0x52, 0x80, 0x91, 0x17, 0x91,
0xb9, 0xd5, 0x24, 0x21, 0x56, 0xfa, 0x88, 0xc1, 0xb5, 0xd6, 0x72, 0x08, 0xeb, 0x66, 0x15, 0x8e,
0x20, 0xb9, 0x7c, 0x52, 0x1f, 0x03, 0xa4, 0xaa, 0x7a, 0x03, 0x5c, 0xd2, 0x9f, 0x19, 0x65, 0xfc,
0x1c, 0xd6, 0xcd, 0x9e, 0xbe, 0x46, 0x40, 0xd3, 0x2d, 0xb9, 0xb4, 0x5b, 0x70, 0x0b, 0xec, 0x03,
0xe2, 0x8a, 0xbb, 0x5a, 0x4e, 0xab, 0x29, 0x91, 0xa6, 0x84, 0xc9, 0xf3, 0x30, 0xe6, 0xc6, 0x44,
0x7c, 0x0b, 0xde, 0x8b, 0x30, 0xe2, 0x12, 0x71, 0x85, 0xc8, 0x6f, 0xfc, 0x25, 0xe4, 0xf7, 0xc2,
0x2e, 0x43, 0x8b, 0x90, 0x73, 0x9b, 0xda, 0x47, 0xce, 0x6d, 0xa2, 0xbb, 0xd2, 0xbd, 0xde, 0x21,
0x95, 0xf4, 0x70, 0x07, 0xc4, 0x25, 0x32, 0xf0, 0x3d, 0xa8, 0xb8, 0xf1, 0x4e, 0x18, 0x46, 0x5d,
0x51, 0xea, 0x30, 0xd2, 0x77, 0xd2, 0x28, 0x13, 0x3f, 0x85, 0xaa, 0x70, 0xdf, 0xe6, 0x94, 0x27,
0x9b, 0x7a, 0x15, 0x8a, 0x82, 0x97, 0x84, 0xd3, 0x94, 0xbc, 0xf7, 0x84, 0x9e, 0x59, 0x80, 0x92,
0xc0, 0x9f, 0x2a, 0x0f, 0xad, 0x53, 0x16, 0xf0, 0x4c, 0x96, 0x24, 0x2d, 0x1d, 0x54, 0x88, 0x22,
0x10, 0x56, 0x47, 0xd1, 0x98, 0x17, 0x53, 0xcc, 0x82, 0x4b, 0xa4, 0x0c, 0xff, 0x68, 0x01, 0x18,
0x40, 0x83, 0x38, 0x31, 0xb1, 0x66, 0x9b, 0xa0, 0x77, 0x33, 0x6f, 0x97, 0xc9, 0x9d, 0x9a, 0x88,
0x48, 0xe6, 0x85, 0xd3, 0x30, 0x2b, 0x54, 0x37, 0x47, 0x35, 0xd5, 0x57, 0x7c, 0x5d, 0x26, 0x71,
0x6d, 0x56, 0x76, 0xfc, 0x41, 0xcc, 0x59, 0xa4, 0x11, 0x89, 0x37, 0x96, 0x62, 0x24, 0xf9, 0x49,
0x19, 0xd3, 0x53, 0x84, 0xee, 0x41, 0x41, 0x20, 0x35, 0x7b, 0x60, 0xfc, 0x18, 0x4a, 0x88, 0xdb,
0xfa, 0x26, 0x99, 0xba, 0x7b, 0x10, 0xe4, 0xe5, 0x8b, 0x5a, 0xb7, 0x8b, 0x7c, 0x4c, 0x57, 0xc1,
0xde, 0xf5, 0x54, 0x7f, 0xdb, 0x44, 0x7c, 0x4a, 0x0e, 0x3d, 0x97, 0xf3, 0x27, 0x38, 0x54, 0xbc,
0x25, 0x96, 0xd5, 0x00, 0x89, 0xbb, 0xe3, 0x26, 0xf7, 0x9b, 0x79, 0x94, 0xda, 0x99, 0x47, 0x69,
0x1b, 0x96, 0xd5, 0x90, 0xfc, 0x97, 0x4e, 0x7f, 0xce, 0xc1, 0x32, 0x61, 0xb1, 0xf7, 0x8a, 0xb9,
0x41, 0xcc, 0xa3, 0x41, 0xb2, 0xe0, 0x3e, 0x0e, 0x8f, 0x74, 0xaa, 0x6d, 0xa2, 0x88, 0xd7, 0xe9,
0x24, 0xf4, 0x10, 0xca, 0xe3, 0xdd, 0x3f, 0xa9, 0x9a, 0x55, 0x41, 0x0f, 0x61, 0xae, 0x1d, 0x0e,
0xa2, 0x4e, 0x72, 0x0d, 0xae, 0xa6, 0xda, 0x0a, 0x99, 0x12, 0x13, 0xa3, 0x96, 0xe9, 0xa3, 0xc2,
0xe5, 0x7d, 0x84, 0x9e, 0x8c, 0xf5, 0x91, 0xfc, 0x73, 0x29, 0x6f, 0xfe, 0x3f, 0x35, 0x18, 0x11,
0x93, 0x51, 0x6d, 0xfc, 0xbd, 0x05, 0x0b, 0x59, 0x08, 0xaf, 0x35, 0x18, 0x49, 0x45, 0x72, 0x53,
0x2b, 0x62, 0x4f, 0xab, 0x48, 0x3e, 0xad, 0x48, 0xfa, 0xce, 0x2d, 0x64, 0xde, 0xb9, 0xf8, 0x04,
0xd6, 0x26, 0xca, 0xb4, 0x13, 0xf6, 0xfa, 0xa2, 0x1f, 0xfe, 0x45, 0xb9, 0xc4, 0xca, 0x88, 0x22,
0x5d, 0xa8, 0x12, 0x51, 0x04, 0x7e, 0x0c, 0xb7, 0xdb, 0x8c, 0x67, 0x8a, 0x64, 0xba, 0xad, 0x0e,
0xf6, 0x1e, 0x3b, 0x9b, 0x71, 0x7c, 0x21, 0xc2, 0x1f, 0x82, 0x73, 0xd0, 0xef, 0x52, 0xce, 0x6e,
0x64, 0xbd, 0x0d, 0xf3, 0xfb, 0x61, 0x3f, 0xf4, 0xc3, 0xe3, 0xe1, 0x15, 0x23, 0xef, 0xc0, 0x9c,
0xda, 0x8f, 0xea, 0x91, 0x52, 0x22, 0x86, 0xc4, 0xb7, 0x44, 0x43, 0x77, 0xa8, 0xdf, 0x19, 0xf8,
0x02, 0x86, 0xf8, 0xf7, 0x8a, 0xb7, 0xab, 0xbf, 0x5e, 0xd4, 0xac, 0xdf, 0x2e, 0x6a, 0xd6, 0x1f,
0x17, 0x35, 0xeb, 0xa7, 0x3f, 0x6b, 0xff, 0x3b, 0x2a, 0xca, 0xbf, 0xf0, 0x47, 0xff, 0x04, 0x00,
0x00, 0xff, 0xff, 0xc3, 0xb3, 0xdc, 0xe3, 0x96, 0x0f, 0x00, 0x00,
}

1
internal/private.proto
View file

@ -3,7 +3,6 @@ syntax = "proto3";
package internal;
message IndexMeta {
string TimeQuantum = 2;
}
message FrameMeta {

142
internal/public.pb.go
View file

@ -11,7 +11,7 @@
It has these top-level messages:
Bitmap
Pair
SumCount
ValCount
Bit
ColumnAttrSet
Attr
@ -105,24 +105,24 @@ func (m *Pair) GetCount() uint64 {
return 0
}
type SumCount struct {
Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"`
type ValCount struct {
Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"`
Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"`
}
func (m *SumCount) Reset() { *m = SumCount{} }
func (m *SumCount) String() string { return proto.CompactTextString(m) }
func (*SumCount) ProtoMessage() {}
func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
func (m *ValCount) Reset() { *m = ValCount{} }
func (m *ValCount) String() string { return proto.CompactTextString(m) }
func (*ValCount) ProtoMessage() {}
func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
func (m *SumCount) GetSum() int64 {
func (m *ValCount) GetVal() int64 {
if m != nil {
return m.Sum
return m.Val
}
return 0
}
func (m *SumCount) GetCount() int64 {
func (m *ValCount) GetCount() int64 {
if m != nil {
return m.Count
}
@ -358,7 +358,7 @@ type QueryResult struct {
Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"`
N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"`
SumCount *SumCount `protobuf:"bytes,5,opt,name=SumCount" json:"SumCount,omitempty"`
ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"`
Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"`
}
@ -395,9 +395,9 @@ func (m *QueryResult) GetPairs() []*Pair {
return nil
}
func (m *QueryResult) GetSumCount() *SumCount {
func (m *QueryResult) GetValCount() *ValCount {
if m != nil {
return m.SumCount
return m.ValCount
}
return nil
}
@ -548,7 +548,7 @@ func (m *ImportValueRequest) GetValues() []int64 {
func init() {
proto.RegisterType((*Bitmap)(nil), "internal.Bitmap")
proto.RegisterType((*Pair)(nil), "internal.Pair")
proto.RegisterType((*SumCount)(nil), "internal.SumCount")
proto.RegisterType((*ValCount)(nil), "internal.ValCount")
proto.RegisterType((*Bit)(nil), "internal.Bit")
proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet")
proto.RegisterType((*Attr)(nil), "internal.Attr")
@ -655,7 +655,7 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *SumCount) Marshal() (dAtA []byte, err error) {
func (m *ValCount) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
@ -665,15 +665,15 @@ func (m *SumCount) Marshal() (dAtA []byte, err error) {
return dAtA[:n], nil
}
func (m *SumCount) MarshalTo(dAtA []byte) (int, error) {
func (m *ValCount) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if m.Sum != 0 {
if m.Val != 0 {
dAtA[i] = 0x8
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Sum))
i = encodeVarintPublic(dAtA, i, uint64(m.Val))
}
if m.Count != 0 {
dAtA[i] = 0x10
@ -1023,11 +1023,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.SumCount != nil {
if m.ValCount != nil {
dAtA[i] = 0x2a
i++
i = encodeVarintPublic(dAtA, i, uint64(m.SumCount.Size()))
n6, err := m.SumCount.MarshalTo(dAtA[i:])
i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size()))
n6, err := m.ValCount.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
@ -1317,11 +1317,11 @@ func (m *Pair) Size() (n int) {
return n
}
func (m *SumCount) Size() (n int) {
func (m *ValCount) Size() (n int) {
var l int
_ = l
if m.Sum != 0 {
n += 1 + sovPublic(uint64(m.Sum))
if m.Val != 0 {
n += 1 + sovPublic(uint64(m.Val))
}
if m.Count != 0 {
n += 1 + sovPublic(uint64(m.Count))
@ -1471,8 +1471,8 @@ func (m *QueryResult) Size() (n int) {
if m.Changed {
n += 2
}
if m.SumCount != nil {
l = m.SumCount.Size()
if m.ValCount != nil {
l = m.ValCount.Size()
n += 1 + l + sovPublic(uint64(l))
}
if m.Type != 0 {
@ -1874,7 +1874,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *SumCount) Unmarshal(dAtA []byte) error {
func (m *ValCount) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
@ -1897,17 +1897,17 @@ func (m *SumCount) Unmarshal(dAtA []byte) error {
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: SumCount: wiretype end group for non-group")
return fmt.Errorf("proto: ValCount: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: SumCount: illegal tag %d (wire type %d)", fieldNum, wire)
return fmt.Errorf("proto: ValCount: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Val", wireType)
}
m.Sum = 0
m.Val = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
@ -1917,7 +1917,7 @@ func (m *SumCount) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.Sum |= (int64(b) & 0x7F) << shift
m.Val |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -2959,7 +2959,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error {
m.Changed = bool(v != 0)
case 5:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field SumCount", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ValCount", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
@ -2983,10 +2983,10 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.SumCount == nil {
m.SumCount = &SumCount{}
if m.ValCount == nil {
m.ValCount = &ValCount{}
}
if err := m.SumCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
if err := m.ValCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
@ -3821,47 +3821,47 @@ var fileDescriptorPublic = []byte{
// 705 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40,
0x14, 0x65, 0x62, 0x27, 0x71, 0x6e, 0x92, 0xaa, 0x1a, 0x41, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85,
0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x29, 0x65, 0xed, 0xb6, 0xa3,
0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x2d, 0x61, 0xed, 0xb6, 0xa3,
0x62, 0xc9, 0x2f, 0xec, 0xb1, 0xda, 0x7c, 0x07, 0x1b, 0x3e, 0x81, 0x8f, 0x60, 0xc5, 0x0a, 0x76,
0x7c, 0x02, 0x94, 0x1f, 0x41, 0xf7, 0x8e, 0x27, 0x76, 0x5a, 0x09, 0x58, 0xb0, 0x9b, 0x73, 0xce,
0xcc, 0xf5, 0x9c, 0xb9, 0xe7, 0x26, 0x30, 0xca, 0xab, 0xb3, 0x38, 0x3a, 0x9f, 0xe6, 0x45, 0xa6,
0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x29, 0xf4, 0xe6, 0x91, 0x4a, 0xc2, 0x9c,
0x73, 0xb0, 0xe7, 0x91, 0x2a, 0x5d, 0xe6, 0x59, 0x81, 0x2d, 0x68, 0xcd, 0x9f, 0x40, 0xf7, 0x85,
0x52, 0x45, 0xe9, 0x76, 0x3c, 0x2b, 0x18, 0xce, 0xb6, 0xa6, 0xe6, 0xdc, 0x14, 0x69, 0xa1, 0x45,
0x3c, 0x79, 0x24, 0x57, 0xa5, 0x6b, 0x79, 0x56, 0x30, 0x10, 0xb4, 0xf6, 0x9f, 0x81, 0xfd, 0x2a,
0x8c, 0x0a, 0xbe, 0x05, 0x9d, 0xc5, 0xbe, 0xcb, 0x3c, 0x16, 0xd8, 0xa2, 0xb3, 0xd8, 0xe7, 0xf7,
0xa1, 0xbb, 0x97, 0x55, 0xa9, 0x72, 0x3b, 0x44, 0x69, 0xc0, 0xb7, 0xc1, 0x3a, 0x92, 0x2b, 0xd7,
0xf2, 0x58, 0x30, 0x10, 0xb8, 0xf4, 0x67, 0xe0, 0x2c, 0xab, 0x64, 0xad, 0x2e, 0xab, 0x84, 0x8a,
0x58, 0x02, 0x97, 0x9b, 0x55, 0xac, 0xba, 0x8a, 0xff, 0x06, 0xac, 0x79, 0xa4, 0x50, 0x14, 0xd9,
0xd5, 0xfa, 0xab, 0x1a, 0xf0, 0x47, 0xe0, 0xec, 0x65, 0x71, 0x95, 0xa4, 0x8b, 0xfd, 0xfa, 0xdb,
0x6b, 0xcc, 0x1f, 0xc3, 0xe0, 0x24, 0x4a, 0x64, 0xa9, 0xc2, 0x24, 0xa7, 0x4b, 0x58, 0xa2, 0x21,
0xfc, 0xb7, 0x30, 0xd6, 0x3b, 0xd1, 0xed, 0x52, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a,
0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xac, 0x72, 0x59, 0xdf, 0x94,
0xd6, 0xdc, 0x83, 0xe1, 0x52, 0x15, 0x51, 0x7a, 0x79, 0x1a, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29,
0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x12, 0x7a, 0xf3, 0x48, 0x25, 0x61, 0xce,
0x39, 0xd8, 0xf3, 0x48, 0x95, 0x2e, 0xf3, 0xac, 0xc0, 0x16, 0xb4, 0xe6, 0x4f, 0xa0, 0xfb, 0x42,
0xa9, 0xa2, 0x74, 0x3b, 0x9e, 0x15, 0x0c, 0x67, 0x5b, 0x53, 0x73, 0x6e, 0x8a, 0xb4, 0xd0, 0x22,
0x9e, 0x3c, 0x92, 0xab, 0xd2, 0xb5, 0x3c, 0x2b, 0x18, 0x08, 0x5a, 0xfb, 0xcf, 0xc0, 0x7e, 0x15,
0x46, 0x05, 0xdf, 0x82, 0xce, 0x62, 0xdf, 0x65, 0x1e, 0x0b, 0x6c, 0xd1, 0x59, 0xec, 0xf3, 0xfb,
0xd0, 0xdd, 0xcb, 0xaa, 0x54, 0xb9, 0x1d, 0xa2, 0x34, 0xe0, 0xdb, 0x60, 0x1d, 0xc9, 0x95, 0x6b,
0x79, 0x2c, 0x18, 0x08, 0x5c, 0xfa, 0x33, 0x70, 0x96, 0x61, 0xbc, 0x56, 0x97, 0x61, 0x4c, 0x45,
0x2c, 0x81, 0xcb, 0xcd, 0x2a, 0x56, 0x5d, 0xc5, 0x7f, 0x03, 0xd6, 0x3c, 0x52, 0x28, 0x8a, 0xec,
0x6a, 0xfd, 0x55, 0x0d, 0xf8, 0x23, 0x70, 0xf6, 0xb2, 0xb8, 0x4a, 0xd2, 0xc5, 0x7e, 0xfd, 0xed,
0x35, 0xe6, 0x8f, 0x61, 0x70, 0x1a, 0x25, 0xb2, 0x54, 0x61, 0x92, 0xd3, 0x25, 0x2c, 0xd1, 0x10,
0xfe, 0x5b, 0x18, 0xeb, 0x9d, 0xe8, 0xf6, 0x44, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a,
0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xae, 0x72, 0x59, 0xdf, 0x94,
0xd6, 0xdc, 0x83, 0xe1, 0x89, 0x2a, 0xa2, 0xf4, 0x72, 0x19, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29,
0xf4, 0xb8, 0x48, 0x95, 0x96, 0x6d, 0xb2, 0xb1, 0xc6, 0xe8, 0x71, 0x9e, 0x65, 0xb1, 0x16, 0xbb,
0x1e, 0x0b, 0x1c, 0xd1, 0x10, 0x7c, 0x02, 0x70, 0x18, 0x67, 0x61, 0x7d, 0xb6, 0xe7, 0xb1, 0x80,
0x89, 0x16, 0xe3, 0xef, 0x42, 0x1f, 0x6f, 0xfa, 0x32, 0xcc, 0x1b, 0xb7, 0xec, 0x0f, 0x6e, 0xfd,
0xcf, 0x0c, 0x46, 0xaf, 0x2b, 0x59, 0xac, 0x84, 0x7c, 0x5f, 0xc9, 0x92, 0xba, 0x42, 0xb8, 0x76,
0xa9, 0x01, 0xdf, 0x81, 0xde, 0x32, 0x8e, 0xce, 0xa5, 0x7e, 0x3b, 0x5b, 0xd4, 0x08, 0xbd, 0x36,
0x6f, 0x5e, 0x92, 0x57, 0x47, 0xb4, 0x29, 0x3c, 0x29, 0x64, 0x92, 0x29, 0x63, 0xa6, 0x46, 0xdc,
0x87, 0xd1, 0xc1, 0xf5, 0x79, 0x5c, 0x5d, 0x48, 0x7d, 0xb4, 0x47, 0xea, 0x06, 0x87, 0xd5, 0x6b,
0x4c, 0x89, 0xef, 0xeb, 0xea, 0x2d, 0xca, 0xff, 0xc0, 0x60, 0x5c, 0x5f, 0xbf, 0xcc, 0xb3, 0xb4,
0x94, 0xd8, 0xa3, 0x83, 0xa2, 0x30, 0x3d, 0x3a, 0x28, 0x0a, 0xbe, 0x0b, 0x7d, 0x21, 0xcb, 0x2a,
0x56, 0xa6, 0xf1, 0x0f, 0x9a, 0xa7, 0x30, 0x67, 0xab, 0x58, 0x09, 0xb3, 0x8b, 0x3f, 0x87, 0xad,
0x8d, 0x20, 0xe9, 0x89, 0x19, 0xce, 0x1e, 0x36, 0xe7, 0x36, 0x74, 0x71, 0x6b, 0xbb, 0xff, 0x8d,
0xc1, 0xb0, 0x55, 0x99, 0x07, 0x66, 0x78, 0xe9, 0x5a, 0xc3, 0xd9, 0x76, 0x53, 0x48, 0xf3, 0xc2,
0x0c, 0xf7, 0x08, 0xd8, 0x71, 0x1d, 0x26, 0x76, 0x8c, 0x2d, 0xc4, 0xe1, 0x34, 0xdf, 0x6f, 0xb5,
0x10, 0x69, 0xa1, 0x45, 0xee, 0x42, 0x7f, 0xef, 0x5d, 0x98, 0x5e, 0xca, 0x0b, 0x0a, 0x93, 0x23,
0x0c, 0xe4, 0xd3, 0x66, 0x38, 0xe9, 0xf5, 0x87, 0x33, 0xde, 0x94, 0x30, 0x8a, 0x68, 0x06, 0xd8,
0xa4, 0x19, 0x7b, 0x31, 0xd6, 0x69, 0xf6, 0x7f, 0x32, 0x18, 0x2f, 0x92, 0x3c, 0x2b, 0x54, 0x2b,
0x21, 0x8b, 0xf4, 0x42, 0x5e, 0x9b, 0x84, 0x10, 0x40, 0xf6, 0xb0, 0x08, 0x13, 0x3d, 0x0a, 0x03,
0xa1, 0x01, 0xb2, 0x94, 0x14, 0x4a, 0x86, 0x2d, 0x34, 0xa0, 0x4c, 0xe0, 0xb0, 0x97, 0xae, 0xad,
0xd3, 0xa4, 0x11, 0x66, 0xdf, 0xcc, 0x7a, 0xe9, 0x76, 0x49, 0x6a, 0x08, 0xcc, 0xfe, 0x7a, 0xd8,
0x31, 0x2f, 0x56, 0x60, 0x89, 0x16, 0x83, 0xef, 0x20, 0xb2, 0x2b, 0xfa, 0x85, 0xeb, 0xd3, 0x2f,
0x9c, 0x81, 0x78, 0x52, 0x97, 0x21, 0xd1, 0x21, 0xb1, 0xc5, 0xf8, 0x5f, 0x18, 0x70, 0xed, 0x91,
0xa6, 0xe8, 0xff, 0x19, 0xc5, 0xbd, 0x91, 0x8c, 0x75, 0x63, 0x70, 0x2f, 0x82, 0xbf, 0xd8, 0xdc,
0x81, 0x1e, 0xdd, 0xc2, 0x58, 0xac, 0xd1, 0x2d, 0x13, 0xfd, 0xdb, 0x26, 0xe6, 0xdb, 0x5f, 0x6f,
0x26, 0xec, 0xfb, 0xcd, 0x84, 0xfd, 0xb8, 0x99, 0xb0, 0x8f, 0xbf, 0x26, 0xf7, 0xce, 0x7a, 0xf4,
0x27, 0xf2, 0xf4, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa3, 0xa0, 0xd2, 0x51, 0x54, 0x06, 0x00,
0xa9, 0x01, 0xdf, 0x81, 0xde, 0x49, 0x1c, 0x9d, 0x4b, 0xfd, 0x76, 0xb6, 0xa8, 0x11, 0x7a, 0x6d,
0xde, 0xbc, 0x24, 0xaf, 0x8e, 0x68, 0x53, 0x78, 0x52, 0xc8, 0x24, 0x53, 0xc6, 0x4c, 0x8d, 0xb8,
0x0f, 0xa3, 0x83, 0xeb, 0xf3, 0xb8, 0xba, 0x90, 0xfa, 0x68, 0x8f, 0xd4, 0x0d, 0x0e, 0xab, 0xd7,
0x98, 0x12, 0xdf, 0xd7, 0xd5, 0x5b, 0x94, 0xff, 0x81, 0xc1, 0xb8, 0xbe, 0x7e, 0x99, 0x67, 0x69,
0x29, 0xb1, 0x47, 0x07, 0x45, 0x61, 0x7a, 0x74, 0x50, 0x14, 0x7c, 0x17, 0xfa, 0x42, 0x96, 0x55,
0xac, 0x4c, 0xe3, 0x1f, 0x34, 0x4f, 0x61, 0xce, 0x56, 0xb1, 0x12, 0x66, 0x17, 0x7f, 0x0e, 0x5b,
0x1b, 0x41, 0xd2, 0x13, 0x33, 0x9c, 0x3d, 0x6c, 0xce, 0x6d, 0xe8, 0xe2, 0xd6, 0x76, 0xff, 0x1b,
0x83, 0x61, 0xab, 0x32, 0x0f, 0xcc, 0xf0, 0xd2, 0xb5, 0x86, 0xb3, 0xed, 0xa6, 0x90, 0xe6, 0x85,
0x19, 0xee, 0x11, 0xb0, 0xe3, 0x3a, 0x4c, 0xec, 0x18, 0x5b, 0x88, 0xc3, 0x69, 0xbe, 0xdf, 0x6a,
0x21, 0xd2, 0x42, 0x8b, 0xdc, 0x85, 0xfe, 0xde, 0xbb, 0x30, 0xbd, 0x94, 0x17, 0x14, 0x26, 0x47,
0x18, 0xc8, 0xa7, 0xcd, 0x70, 0xd2, 0xeb, 0x0f, 0x67, 0xbc, 0x29, 0x61, 0x14, 0xd1, 0x0c, 0xb0,
0x49, 0x33, 0xf6, 0x62, 0xac, 0xd3, 0xec, 0xff, 0x64, 0x30, 0x5e, 0x24, 0x79, 0x56, 0xa8, 0x56,
0x42, 0x16, 0xe9, 0x85, 0xbc, 0x36, 0x09, 0x21, 0x80, 0xec, 0x61, 0x11, 0x26, 0x7a, 0x14, 0x06,
0x42, 0x03, 0x64, 0x29, 0x29, 0x94, 0x0c, 0x5b, 0x68, 0x40, 0x99, 0xc0, 0x61, 0x2f, 0x5d, 0x5b,
0xa7, 0x49, 0x23, 0xcc, 0xbe, 0x99, 0xf5, 0xd2, 0xed, 0x92, 0xd4, 0x10, 0x98, 0xfd, 0xf5, 0xb0,
0x63, 0x5e, 0xac, 0xc0, 0x12, 0x2d, 0x06, 0xdf, 0x41, 0x64, 0x57, 0xf4, 0x0b, 0xd7, 0xa7, 0x5f,
0x38, 0x03, 0xf1, 0xa4, 0x2e, 0x43, 0xa2, 0x43, 0x62, 0x8b, 0xf1, 0xbf, 0x30, 0xe0, 0xda, 0x23,
0x4d, 0xd1, 0xff, 0x33, 0x8a, 0x7b, 0x23, 0x19, 0xeb, 0xc6, 0xe0, 0x5e, 0x04, 0x7f, 0xb1, 0xb9,
0x03, 0x3d, 0xba, 0x85, 0xb1, 0x58, 0xa3, 0x5b, 0x26, 0xfa, 0xb7, 0x4d, 0xcc, 0xb7, 0xbf, 0xde,
0x4c, 0xd8, 0xf7, 0x9b, 0x09, 0xfb, 0x71, 0x33, 0x61, 0x1f, 0x7f, 0x4d, 0xee, 0x9d, 0xf5, 0xe8,
0x4f, 0xe4, 0xe9, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x3e, 0xba, 0x23, 0x54, 0x06, 0x00,
0x00,
}

6
internal/public.proto
View file

@ -14,8 +14,8 @@ message Pair {
uint64 Count = 2;
}
message SumCount {
int64 Sum = 1;
message ValCount {
int64 Val = 1;
int64 Count = 2;
}
@ -64,7 +64,7 @@ message QueryResult {
Bitmap Bitmap = 1;
uint64 N = 2;
repeated Pair Pairs = 3;
SumCount SumCount = 5;
ValCount ValCount = 5;
bool Changed = 4;
}

133
lru/lru.go Normal file
View file

@ -0,0 +1,133 @@
/*
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package lru implements an LRU cache.
package lru
import "container/list"
// Cache is an LRU cache. It is not safe for concurrent access.
type Cache struct {
// MaxEntries is the maximum number of cache entries before
// an item is evicted. Zero means no limit.
MaxEntries int
// OnEvicted optionally specificies a callback function to be
// executed when an entry is purged from the cache.
OnEvicted func(key Key, value interface{})
ll *list.List
cache map[interface{}]*list.Element
}
// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators
type Key interface{}
type entry struct {
key Key
value interface{}
}
// New creates a new Cache.
// If maxEntries is zero, the cache has no limit and it's assumed
// that eviction is done by the caller.
func New(maxEntries int) *Cache {
return &Cache{
MaxEntries: maxEntries,
ll: list.New(),
cache: make(map[interface{}]*list.Element),
}
}
// Add adds a value to the cache.
func (c *Cache) Add(key Key, value interface{}) {
if c.cache == nil {
c.cache = make(map[interface{}]*list.Element)
c.ll = list.New()
}
if ee, ok := c.cache[key]; ok {
c.ll.MoveToFront(ee)
ee.Value.(*entry).value = value
return
}
ele := c.ll.PushFront(&entry{key, value})
c.cache[key] = ele
if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
c.RemoveOldest()
}
}
// Get looks up a key's value from the cache.
func (c *Cache) Get(key Key) (value interface{}, ok bool) {
if c.cache == nil {
return
}
if ele, hit := c.cache[key]; hit {
c.ll.MoveToFront(ele)
return ele.Value.(*entry).value, true
}
return
}
// Remove removes the provided key from the cache.
func (c *Cache) Remove(key Key) {
if c.cache == nil {
return
}
if ele, hit := c.cache[key]; hit {
c.removeElement(ele)
}
}
// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
if c.cache == nil {
return
}
ele := c.ll.Back()
if ele != nil {
c.removeElement(ele)
}
}
func (c *Cache) removeElement(e *list.Element) {
c.ll.Remove(e)
kv := e.Value.(*entry)
delete(c.cache, kv.key)
if c.OnEvicted != nil {
c.OnEvicted(kv.key, kv.value)
}
}
// Len returns the number of items in the cache.
func (c *Cache) Len() int {
if c.cache == nil {
return 0
}
return c.ll.Len()
}
// Clear purges all stored items from the cache.
func (c *Cache) Clear() {
if c.OnEvicted != nil {
for _, e := range c.cache {
kv := e.Value.(*entry)
c.OnEvicted(kv.key, kv.value)
}
}
c.ll = nil
c.cache = nil
}

42
pilosa.go
View file

@ -46,19 +46,16 @@ var (
ErrInputDefinitionActionRequired = errors.New("field definitions require an action")
ErrInputDefinitionNotFound = errors.New("input-definition not found")
ErrFieldNotFound = errors.New("field not found")
ErrFieldExists = errors.New("field already exists")
ErrFieldNameRequired = errors.New("field name required")
ErrInvalidFieldType = errors.New("invalid field type")
ErrInvalidFieldRange = errors.New("invalid field range")
ErrInverseRangeNotAllowed = errors.New("inverse range not allowed")
ErrRangeCacheNotAllowed = errors.New("range cache not allowed")
ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed")
ErrInvalidFieldValueType = errors.New("invalid field value type")
ErrFieldValueTooLow = errors.New("field value too low")
ErrFieldValueTooHigh = errors.New("field value too high")
ErrInvalidRangeOperation = errors.New("invalid range operation")
ErrInvalidBetweenValue = errors.New("invalid value for between operation")
ErrFieldNotFound = errors.New("field not found")
ErrFieldExists = errors.New("field already exists")
ErrFieldNameRequired = errors.New("field name required")
ErrInvalidFieldType = errors.New("invalid field type")
ErrInvalidFieldRange = errors.New("invalid field range")
ErrInvalidFieldValueType = errors.New("invalid field value type")
ErrFieldValueTooLow = errors.New("field value too low")
ErrFieldValueTooHigh = errors.New("field value too high")
ErrInvalidRangeOperation = errors.New("invalid range operation")
ErrInvalidBetweenValue = errors.New("invalid value for between operation")
ErrInvalidView = errors.New("invalid view")
ErrInvalidCacheType = errors.New("invalid cache type")
@ -71,9 +68,26 @@ var (
ErrQueryRequired = errors.New("query required")
ErrTooManyWrites = errors.New("too many write commands")
ErrConfigClusterEnabledHosts = errors.New("providing hosts to a non-disabled cluster is not allowed")
ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice")
ErrNodeIDNotExists = errors.New("node with provided ID does not exist")
ErrNodeNotCoordinator = errors.New("node is not the coordinator")
ErrResizeNotRunning = errors.New("no resize job currently running")
)
// ApiMethodNotAllowedError wraps an error value indicating that a particular
// API method is not allowed in the current cluster state.
type ApiMethodNotAllowedError struct {
error
}
// BadRequestError wraps an error value to signify that a request could not be
// read, decoded, or parsed such that in an HTTP scenario, http.StatusBadRequest
// would be returned.
type BadRequestError struct {
error
}
// Regular expression to validate index and frame names.
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)

52
roaring/roaring.go
View file

@ -1674,19 +1674,6 @@ func (c *container) clone() *container {
return other
}
// flipBitmap returns a new bitmap containter containing the inverse of all
// bits in c.
func (c *container) flipBitmap() *container {
other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
for i, bitmap := range c.bitmap {
other.bitmap[i] = ^bitmap
}
other.n = other.count()
return other
}
// WriteTo writes c to w.
func (c *container) WriteTo(w io.Writer) (n int64, err error) {
if c.isArray() {
@ -1812,6 +1799,43 @@ type ContainerInfo struct {
Pointer unsafe.Pointer // offset within the mmap
}
// flip returns a new container containing the inverse of all
// bits in a.
func flip(a *container) *container {
if a.isArray() {
return flipArray(a)
} else if a.isRun() {
return flipRun(a)
} else {
return flipBitmap(a)
}
}
func flipArray(b *container) *container {
// TODO: actually implement this
x := b.clone()
x.arrayToBitmap()
return flipBitmap(x)
}
func flipBitmap(b *container) *container {
other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
for i, bitmap := range b.bitmap {
other.bitmap[i] = ^bitmap
}
other.n = other.count()
return other
}
func flipRun(b *container) *container {
// TODO: actually implement this
x := b.clone()
x.runToBitmap()
return flipBitmap(x)
}
func intersectionCount(a, b *container) int {
if a.isArray() {
if b.isArray() {
@ -2571,7 +2595,7 @@ RUNLOOP:
func differenceRunBitmap(a, b *container) *container {
// If a is full, difference is the flip of b.
if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 {
return b.flipBitmap()
return flipBitmap(b)
}
output := &container{containerType: ContainerRun}
output.n = a.n

5
roaring/roaring_helpers_test.go
View file

@ -220,8 +220,11 @@ func runEvenBitsSet() []interval16 {
///////////////////////////////////////////////////////////////////////////
// f is a container function taking either one or two containers as input
// func(a *container) *container
// func(a, b *container) *container
type testOp struct {
f func(a, b *container) *container
f interface{}
x string
y string
exp string

57
roaring/roaring_internal_test.go
View file

@ -2020,38 +2020,6 @@ func TestXorRunRun(t *testing.T) {
}
}
func TestBitmapFlip(t *testing.T) {
c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
ttable := []struct {
original uint64
flipped uint64
}{
{0x0000000000000000, 0xFFFFFFFFFFFFFFFF},
{0xFFFFFFFFFFFFFFFF, 0x0000000000000000},
{0xFFFFFFFFFFFFFFF0, 0x000000000000000F},
{0xFFFFFFEFFFFFFFFF, 0x0000001000000000},
{0x0000001000000000, 0xFFFFFFEFFFFFFFFF},
}
expectedN := int(65536)
for i, tt := range ttable {
c.bitmap[i] = tt.original
expectedN -= int(popcount(tt.original))
}
o := c.flipBitmap()
for i, tt := range ttable {
if o.bitmap[i] != tt.flipped {
t.Fatalf("bitmapFlip calculation. expected %v, got %v", tt.flipped, o.bitmap[i])
}
}
if o.n != expectedN {
t.Fatalf("bitmapFlip calculation. expected count %v, got %v", expectedN, o.n)
}
}
func TestBitmapXorRange(t *testing.T) {
c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
tests := []struct {
@ -3185,12 +3153,24 @@ func TestContainerCombinations(t *testing.T) {
//{xor, "evenBitsSet", "outerBitsSet", ""},
{xor, "evenBitsSet", "oddBitsSet", "full"},
{xor, "evenBitsSet", "evenBitsSet", "empty"},
// flip
{flip, "empty", "", "full"},
{flip, "full", "", "empty"},
{flip, "firstBitSet", "", "firstBitUnset"},
{flip, "lastBitSet", "", "lastBitUnset"},
{flip, "firstBitUnset", "", "firstBitSet"},
{flip, "lastBitUnset", "", "lastBitSet"},
{flip, "innerBitsSet", "", "outerBitsSet"},
{flip, "outerBitsSet", "", "innerBitsSet"},
{flip, "oddBitsSet", "", "evenBitsSet"},
{flip, "evenBitsSet", "", "oddBitsSet"},
}
for _, testOp := range testOps {
for _, x := range containerTypes {
for _, y := range containerTypes {
desc := fmt.Sprintf("%s(%s/%s, %s/%s)", getFunctionName(testOp.f), cm[x], testOp.x, cm[y], testOp.y)
ret := testOp.f(cts[x][testOp.x], cts[y][testOp.y])
ret := runContainerFunc(testOp.f, cts[x][testOp.x], cts[y][testOp.y])
exp := testOp.exp
// Convert to all container types and check result.
@ -3240,3 +3220,14 @@ func TestContainerCombinations(t *testing.T) {
}
}
}
//func getFunc(func(a, b *container) *container, m, n *container) *container {
func runContainerFunc(f interface{}, c ...*container) *container {
switch f.(type) {
case func(*container) *container:
return f.(func(*container) *container)(c[0])
case func(*container, *container) *container:
return f.(func(a, b *container) *container)(c[0], c[1])
}
return nil
}

32
security_manager.go
View file

@ -1,32 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
// SecurityManager provides the ability to limit access to restricted endpoints
// during cluster configuration.
type SecurityManager interface {
SetRestricted()
SetNormal()
}
// NopSecurityManager provides a no-op implementation of the SecurityManager interface.
type NopSecurityManager struct {
}
// SetRestricted no-op.
func (sdm *NopSecurityManager) SetRestricted() {}
// SetNormal no-op.
func (sdm *NopSecurityManager) SetNormal() {}

398
server.go
View file

@ -16,9 +16,8 @@ package pilosa
import (
"context"
"crypto/tls"
"errors"
"fmt"
"log"
"net"
"net/http"
"os"
@ -31,14 +30,14 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
// Default server settings.
const (
DefaultAntiEntropyInterval = 10 * time.Minute
DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics"
DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics"
)
// Ensure Server implements interfaces.
@ -48,91 +47,202 @@ var _ StatusHandler = &Server{}
// Server represents a holder wrapped by a running HTTP server.
type Server struct {
ln net.Listener
// Close management.
wg sync.WaitGroup
closing chan struct{}
// Data storage and HTTP interface.
Holder *Holder
Handler *Handler
// Internal
Holder *Holder
Cluster *Cluster
diagnostics *DiagnosticsCollector
executor *Executor
// External
handler *Handler
Broadcaster Broadcaster
BroadcastReceiver BroadcastReceiver
Gossiper Gossiper
RemoteClient *http.Client
remoteClient *http.Client
systemInfo SystemInfo
gcNotifier GCNotifier
NewAttrStore func(string) AttrStore
logger Logger
ln net.Listener
// Cluster configuration.
Network string
NodeID string
URI URI
Cluster *Cluster
diagnostics *DiagnosticsCollector
SystemInfo SystemInfo
GCNotifier GCNotifier
NewAttrStore func(string) AttrStore
// Background monitoring intervals.
AntiEntropyInterval time.Duration
MetricInterval time.Duration
DiagnosticInterval time.Duration
// TLS configuration
TLS *tls.Config
// Misc options.
MaxWritesPerRequest int
Logger Logger
NodeID string
URI URI
antiEntropyInterval time.Duration
metricInterval time.Duration
diagnosticInterval time.Duration
maxWritesPerRequest int
defaultClient InternalClient
}
// NewServer returns a new instance of Server.
func NewServer() *Server {
s := &Server{
closing: make(chan struct{}),
// ServerOption is a functional option type for pilosa.Server
type ServerOption func(s *Server) error
func OptServerLogger(l Logger) ServerOption {
return func(s *Server) error {
s.logger = l
return nil
}
}
func OptServerReplicaN(n int) ServerOption {
return func(s *Server) error {
s.Cluster.ReplicaN = n
return nil
}
}
func OptServerDataDir(dir string) ServerOption {
return func(s *Server) error {
s.Cluster.Path = dir
s.Holder.Path = dir
return nil
}
}
func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption {
return func(s *Server) error {
s.NewAttrStore = af
s.Holder.NewAttrStore = af
return nil
}
}
func OptServerAntiEntropyInterval(interval time.Duration) ServerOption {
return func(s *Server) error {
s.antiEntropyInterval = interval
return nil
}
}
func OptServerLongQueryTime(dur time.Duration) ServerOption {
return func(s *Server) error {
s.Cluster.LongQueryTime = dur
return nil
}
}
func OptServerHandler(h *Handler) ServerOption {
return func(s *Server) error {
s.handler = h
return nil
}
}
func OptServerMaxWritesPerRequest(n int) ServerOption {
return func(s *Server) error {
s.maxWritesPerRequest = n
return nil
}
}
func OptServerMetricInterval(dur time.Duration) ServerOption {
return func(s *Server) error {
s.metricInterval = dur
return nil
}
}
func OptServerSystemInfo(si SystemInfo) ServerOption {
return func(s *Server) error {
s.systemInfo = si
return nil
}
}
func OptServerGCNotifier(gcn GCNotifier) ServerOption {
return func(s *Server) error {
s.gcNotifier = gcn
return nil
}
}
func OptServerRemoteClient(c *http.Client) ServerOption {
return func(s *Server) error {
s.executor = NewExecutor(c)
s.remoteClient = c
s.defaultClient = NewInternalHTTPClientFromURI(nil, c)
s.Cluster.RemoteClient = c
return nil
}
}
func OptServerStatsClient(sc StatsClient) ServerOption {
return func(s *Server) error {
s.Holder.Stats = sc
return nil
}
}
func OptServerDiagnosticsInterval(dur time.Duration) ServerOption {
return func(s *Server) error {
s.diagnosticInterval = dur
return nil
}
}
func OptServerListener(ln net.Listener) ServerOption {
return func(s *Server) error {
s.ln = ln
return nil
}
}
func OptServerURI(uri *URI) ServerOption {
return func(s *Server) error {
s.URI = *uri
return nil
}
}
// NewServer returns a new instance of Server.
func NewServer(opts ...ServerOption) (*Server, error) {
s := &Server{
closing: make(chan struct{}),
Cluster: NewCluster(),
Holder: NewHolder(),
Handler: NewHandler(),
handler: NewHandler(),
Broadcaster: NopBroadcaster,
BroadcastReceiver: NopBroadcastReceiver,
diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer),
SystemInfo: NewNopSystemInfo(),
systemInfo: NewNopSystemInfo(),
Network: "tcp",
GCNotifier: NopGCNotifier,
gcNotifier: NopGCNotifier,
NewAttrStore: NewNopAttrStore,
AntiEntropyInterval: DefaultAntiEntropyInterval,
MetricInterval: 0,
DiagnosticInterval: 0,
antiEntropyInterval: time.Minute * 10,
metricInterval: 0,
diagnosticInterval: 0,
Logger: NopLogger,
logger: NopLogger,
}
s.Handler.Holder = s.Holder
s.diagnostics.server = s
return s
}
// Open opens and initializes the server.
func (s *Server) Open() error {
s.Logger.Printf("open server")
// s.ln can be configured prior to Open() via s.OpenListener().
if s.ln == nil {
if err := s.OpenListener(); err != nil {
return err
for _, opt := range opts {
err := opt(s)
if err != nil {
return nil, errors.Wrap(err, "applying option")
}
}
// Get or create NodeID.
s.NodeID = s.LoadNodeID()
s.Holder.Logger = s.logger
s.Holder.Stats.SetLogger(s.logger)
s.Cluster.Logger = s.logger
s.Cluster.Holder = s.Holder
// update URI port with actual listener port. TODO this should probably be done outside of here.
if s.URI.Port() == 0 {
s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port))
}
s.NodeID = s.LoadNodeID()
// Set Cluster Node.
node := &Node{
ID: s.NodeID,
@ -140,47 +250,59 @@ func (s *Server) Open() error {
IsCoordinator: s.Cluster.Coordinator == s.NodeID,
}
s.Cluster.Node = node
// Append the NodeID tag to stats.
s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%s", s.NodeID))
// Peek at the holder to determine if there is data on disk.
// Don't actually load the data until after the Cluster
// management starts.
s.Holder.Peek()
s.executor.Holder = s.Holder
s.executor.Node = node
s.executor.Cluster = s.Cluster
s.executor.MaxWritesPerRequest = s.maxWritesPerRequest
s.handler.API.Executor = s.executor
return s, nil
}
// Open opens and initializes the server.
func (s *Server) Open() error {
s.logger.Printf("open server")
// s.ln can be configured prior to Open() via s.OpenListener().
if s.ln == nil {
return errors.New("Must pass a listener option to NewServer")
}
// Log startup
err := s.Holder.logStartup()
if err != nil {
log.Println(errors.Wrap(err, "logging startup"))
}
// Get or create NodeID.
// Append the NodeID tag to stats.
// Create default HTTP client
s.createDefaultClient(s.RemoteClient)
// Create executor for executing queries.
e := NewExecutor(s.RemoteClient)
e.Holder = s.Holder
e.Node = node
e.Cluster = s.Cluster
e.MaxWritesPerRequest = s.MaxWritesPerRequest
// Cluster settings.
s.Cluster.Broadcaster = s.Broadcaster
s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest
s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest
// Initialize HTTP handler.
s.Handler.Broadcaster = s.Broadcaster
s.Handler.BroadcastHandler = s
s.Handler.StatusHandler = s
s.Handler.Node = node
s.Handler.Cluster = s.Cluster
s.Handler.Executor = e
s.Cluster.prefect = s.Handler
s.handler.API.Holder = s.Holder
s.handler.API.Broadcaster = s.Broadcaster
s.handler.API.BroadcastHandler = s
s.handler.API.StatusHandler = s
s.handler.API.URI = s.URI
s.handler.API.Cluster = s.Cluster
// Initialize Holder.
s.Holder.Broadcaster = s.Broadcaster
// Serve HTTP.
go func() {
err := http.Serve(s.ln, s.Handler)
err := http.Serve(s.ln, s.handler)
if err != nil {
s.Logger.Printf("HTTP handler terminated with error: %s\n", err)
s.logger.Printf("HTTP handler terminated with error: %s\n", err)
}
}()
@ -218,43 +340,6 @@ func (s *Server) Open() error {
return nil
}
// OpenListener opens a listener for the Server.
func (s *Server) OpenListener() error {
s.Logger.Printf("open server listener: %s", s.URI)
if s.ln != nil {
return fmt.Errorf("a listener already exists for server: %s", s.URI)
}
var ln net.Listener
var err error
// If bind URI has the https scheme, enable TLS
if s.URI.Scheme() == "https" && s.TLS != nil {
ln, err = tls.Listen("tcp", s.URI.HostPort(), s.TLS)
if err != nil {
return err
}
} else if s.URI.Scheme() == "http" {
// Open HTTP listener to determine port (if specified as :0).
ln, err = net.Listen(s.Network, s.URI.HostPort())
if err != nil {
return fmt.Errorf("net.Listen: %v", err)
}
} else {
return fmt.Errorf("unsupported scheme: %s", s.URI.Scheme())
}
s.ln = ln
if s.URI.Port() == 0 {
// If the port is 0, it is set automatically.
// Find out automatically set port and update the host.
s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port))
}
return nil
}
// Close closes the server and waits for it to shutdown.
func (s *Server) Close() error {
// Notify goroutines to stop.
@ -282,7 +367,7 @@ func (s *Server) LoadNodeID() string {
}
nodeID, err := s.Holder.loadNodeID()
if err != nil {
s.Logger.Printf("loading NodeID: %v", err)
s.logger.Printf("loading NodeID: %v", err)
return s.NodeID
}
return nodeID
@ -295,31 +380,12 @@ func (s *Server) Addr() net.Addr {
}
return s.ln.Addr()
}
func GetHTTPClient(t *tls.Config) *http.Client {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
MaxIdleConns: 1000,
MaxIdleConnsPerHost: 200,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
if t != nil {
transport.TLSClientConfig = t
}
return &http.Client{Transport: transport}
}
func (s *Server) monitorAntiEntropy() {
ticker := time.NewTicker(s.AntiEntropyInterval)
ticker := time.NewTicker(s.antiEntropyInterval)
defer ticker.Stop()
s.Logger.Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval)
s.logger.Printf("holder sync monitor initializing (%s interval)", s.antiEntropyInterval)
for {
// Wait for tick or a close.
@ -330,7 +396,7 @@ func (s *Server) monitorAntiEntropy() {
s.Holder.Stats.Count("AntiEntropy", 1, 1.0)
}
t := time.Now()
s.Logger.Printf("holder sync beginning")
s.logger.Printf("holder sync beginning")
// Initialize syncer with local holder and remote client.
var syncer HolderSyncer
@ -338,17 +404,17 @@ func (s *Server) monitorAntiEntropy() {
syncer.Node = s.Cluster.Node
syncer.Cluster = s.Cluster
syncer.Closing = s.closing
syncer.RemoteClient = s.RemoteClient
syncer.RemoteClient = s.remoteClient
syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer")
// Sync holders.
if err := syncer.SyncHolder(); err != nil {
s.Logger.Printf("holder sync error: err=%s", err)
s.logger.Printf("holder sync error: err=%s", err)
continue
}
// Record successful sync in log.
s.Logger.Printf("holder sync complete")
s.logger.Printf("holder sync complete")
dif := time.Since(t)
s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0)
}
@ -368,9 +434,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
idx.SetRemoteMaxSlice(obj.Slice)
}
case *internal.CreateIndexMessage:
opt := IndexOptions{
TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum),
}
opt := IndexOptions{}
_, err := s.Holder.CreateIndex(obj.Index, opt)
if err != nil {
return err
@ -472,7 +536,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
func (s *Server) SendSync(pb proto.Message) error {
var eg errgroup.Group
for _, node := range s.Cluster.Nodes {
s.Logger.Printf("SendSync to: %s", node.URI)
s.logger.Printf("SendSync to: %s", node.URI)
// Don't forward the message to ourselves.
if s.URI == node.URI {
continue
@ -494,7 +558,7 @@ func (s *Server) SendAsync(pb proto.Message) error {
// SendTo represents an implementation of Broadcaster.
func (s *Server) SendTo(to *Node, pb proto.Message) error {
s.Logger.Printf("SendTo: %s", to.URI)
s.logger.Printf("SendTo: %s", to.URI)
ctx := context.WithValue(context.Background(), "uri", &to.URI)
return s.defaultClient.SendMessage(ctx, pb)
}
@ -544,7 +608,7 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error {
err := s.mergeRemoteStatus(pb.(*internal.NodeStatus))
if err != nil {
s.Logger.Printf("merge remote status: %s", err)
s.logger.Printf("merge remote status: %s", err)
}
}()
@ -569,7 +633,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex == nil {
s.Logger.Printf("Local Index not found: %s", index)
s.logger.Printf("Local Index not found: %s", index)
continue
}
if newMax > oldmaxslices[index] {
@ -585,7 +649,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex == nil {
s.Logger.Printf("Local Index not found: %s", index)
s.logger.Printf("Local Index not found: %s", index)
continue
}
if newMaxInverse > oldMaxInverseSlices[index] {
@ -600,14 +664,14 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
// monitorDiagnostics periodically polls the Pilosa Indexes for cluster info.
func (s *Server) monitorDiagnostics() {
// Do not send more than once a minute
if s.DiagnosticInterval < time.Minute {
s.Logger.Printf("diagnostics disabled")
if s.diagnosticInterval < time.Minute {
s.logger.Printf("diagnostics disabled")
return
} else {
s.Logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval)
s.logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.diagnosticInterval)
}
s.diagnostics.Logger = s.Logger
s.diagnostics.Logger = s.logger
s.diagnostics.SetVersion(Version)
s.diagnostics.Set("Host", s.URI.host)
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ","))
@ -629,11 +693,11 @@ func (s *Server) monitorDiagnostics() {
s.diagnostics.CheckVersion()
err = s.diagnostics.Flush()
if err != nil {
s.Logger.Printf("Diagnostics error: %s", err)
s.logger.Printf("Diagnostics error: %s", err)
}
}
ticker := time.NewTicker(s.DiagnosticInterval)
ticker := time.NewTicker(s.diagnosticInterval)
defer ticker.Stop()
flush()
for {
@ -650,24 +714,24 @@ func (s *Server) monitorDiagnostics() {
// monitorRuntime periodically polls the Go runtime metrics.
func (s *Server) monitorRuntime() {
// Disable metrics when poll interval is zero.
if s.MetricInterval <= 0 {
if s.metricInterval <= 0 {
return
}
var m runtime.MemStats
ticker := time.NewTicker(s.MetricInterval)
ticker := time.NewTicker(s.metricInterval)
defer ticker.Stop()
defer s.GCNotifier.Close()
defer s.gcNotifier.Close()
s.Logger.Printf("runtime stats initializing (%s interval)", s.MetricInterval)
s.logger.Printf("runtime stats initializing (%s interval)", s.metricInterval)
for {
// Wait for tick or a close.
select {
case <-s.closing:
return
case <-s.GCNotifier.AfterGC():
case <-s.gcNotifier.AfterGC():
// GC just ran.
s.Holder.Stats.Count("garbage_collection", 1, 1.0)
case <-ticker.C:
@ -692,10 +756,6 @@ func (s *Server) monitorRuntime() {
}
}
func (s *Server) createDefaultClient(remoteClient *http.Client) {
s.defaultClient = NewInternalHTTPClientFromURI(nil, remoteClient)
}
// CountOpenFiles on operating systems that support lsof.
func CountOpenFiles() (int, error) {
switch runtime.GOOS {

84
server/cluster_test.go
View file

@ -26,80 +26,18 @@ import (
"golang.org/x/sync/errgroup"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/test"
)
// Ensure program can send/receive broadcast messages.
func TestMain_SendReceiveMessage(t *testing.T) {
m0 := test.MustRunMain()
ms := test.MustRunMainWithCluster(t, 2)
m0, m1 := ms[0], ms[1]
defer m0.Close()
m1 := test.MustRunMain()
defer m1.Close()
// Update cluster config
m0.Server.Cluster.Nodes = []*pilosa.Node{
{ID: m0.Server.NodeID, URI: m0.Server.URI},
{ID: m1.Server.NodeID, URI: m1.Server.URI},
}
m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes
// Configure node0
// get the host portion of addr to use for binding
m0.Config.Gossip.Port = "0"
m0.Config.Gossip.Seeds = []string{}
m0.Server.Cluster.Coordinator = m0.Server.NodeID
m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}}
m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.Logger)
gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server)
if err != nil {
t.Fatal(err)
}
m0.Server.Cluster.MemberSet = gossipMemberSet0
m0.Server.Broadcaster = m0.Server
m0.Server.Gossiper = gossipMemberSet0
m0.Server.Handler.Broadcaster = m0.Server.Broadcaster
m0.Server.Holder.Broadcaster = m0.Server.Broadcaster
m0.Server.BroadcastReceiver = gossipMemberSet0
if err := m0.Server.BroadcastReceiver.Start(m0.Server); err != nil {
t.Fatal(err)
}
// Open Cluster management.
if err := m0.Server.Cluster.Open(); err != nil {
t.Fatal(err)
}
// Configure node1
// get the host portion of addr to use for binding
m1.Config.Gossip.Port = "0"
m1.Config.Gossip.Seeds = []string{gossipMemberSet0.GetBindAddr()}
m1.Server.Cluster.Coordinator = m0.Server.NodeID
m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.Logger)
gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server)
if err != nil {
t.Fatal(err)
}
m1.Server.Cluster.MemberSet = gossipMemberSet1
m1.Server.Broadcaster = m1.Server
m1.Server.Gossiper = gossipMemberSet1
m1.Server.Handler.Broadcaster = m1.Server.Broadcaster
m1.Server.Holder.Broadcaster = m1.Server.Broadcaster
m1.Server.BroadcastReceiver = gossipMemberSet1
if err := m1.Server.BroadcastReceiver.Start(m1.Server); err != nil {
t.Fatal(err)
}
// Open Cluster management.
if err := m1.Server.Cluster.Open(); err != nil {
t.Fatal(err)
}
m0.Server.Cluster.SetState(pilosa.ClusterStateNormal)
m1.Server.Cluster.SetState(pilosa.ClusterStateNormal)
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -458,7 +396,7 @@ func TestCluster_GossipMembership(t *testing.T) {
var eg errgroup.Group
eg.Go(func() error {
// Pass invalid seed as first in list
_, err = m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed})
_, err := m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed})
if err != nil {
return err
}
@ -471,7 +409,7 @@ func TestCluster_GossipMembership(t *testing.T) {
eg.Go(func() error {
// Pass invalid seed as last in list
_, err = m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"})
_, err := m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"})
if err != nil {
return err
}
@ -507,9 +445,9 @@ func TestClusterResize_RemoveNode(t *testing.T) {
t.Run("ErrorRemoveInvalidNode", func(t *testing.T) {
resp := test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), `{"id": "invalid-node-id"}`)
expBody := "Node is not a member of the cluster: invalid-node-id"
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected StatusCode %d but got %d", http.StatusBadRequest, resp.StatusCode)
expBody := "removing node: finding node to remove: node with provided ID does not exist"
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("expected StatusCode %d but got %d", http.StatusNotFound, resp.StatusCode)
} else if strings.TrimSpace(resp.Body) != expBody {
t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body))
}
@ -521,7 +459,7 @@ func TestClusterResize_RemoveNode(t *testing.T) {
resp = test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID))
expBody := "The coordinator node cannot be removed. First, make a different node the new coordinator."
expBody := "removing node: calling node leave: coordinator cannot be removed; first, make a different node the new coordinator."
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode)
} else if strings.TrimSpace(resp.Body) != expBody {
@ -538,7 +476,7 @@ func TestClusterResize_RemoveNode(t *testing.T) {
resp = test.MustDo("POST", m1.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID))
expBody := fmt.Sprintf("Node removal requests are only valid on the Coordinator node: %s", coordinatorNodeID)
expBody := fmt.Sprintf("removing node: calling node leave: node removal requests are only valid on the coordinator node: %s", coordinatorNodeID)
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode)
} else if strings.TrimSpace(resp.Body) != expBody {

132
server/config.go Normal file
View file

@ -0,0 +1,132 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package server
import (
"time"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/toml"
)
// Cluster types.
const (
ClusterNone = ""
ClusterStatic = "static"
ClusterGossip = "gossip"
)
// TLSConfig contains TLS configuration
type TLSConfig struct {
// CertificatePath contains the path to the certificate (.crt or .pem file)
CertificatePath string `toml:"certificate-path"`
// CertificateKeyPath contains the path to the certificate key (.key file)
CertificateKeyPath string `toml:"certificate-key-path"`
// SkipVerify disables verification for self-signed certificates
SkipVerify bool `toml:"skip-verify"`
}
// Config represents the configuration for the command.
type Config struct {
// DataDir is the directory where Pilosa stores both indexed data and
// running state such as cluster topology information.
DataDir string `toml:"data-dir"`
// Bind is the host:port on which Pilosa will listen.
Bind string `toml:"bind"`
// MaxWritesPerRequest limits the number of mutating commands that can be in
// a single request to the server. This includes SetBit, ClearBit,
// SetRowAttrs & SetColumnAttrs.
MaxWritesPerRequest int `toml:"max-writes-per-request"`
// LogPath configures where Pilosa will write logs.
LogPath string `toml:"log-path"`
// Verbose toggles verbose logging which can be useful for debugging.
Verbose bool `toml:"verbose"`
// TLS
TLS TLSConfig
Cluster struct {
// Disabled controls whether clustering functionality is enabled.
Disabled bool `toml:"disabled"`
Coordinator bool `toml:"coordinator"`
ReplicaN int `toml:"replicas"`
Hosts []string `toml:"hosts"`
LongQueryTime toml.Duration `toml:"long-query-time"`
} `toml:"cluster"`
// Gossip config is based around memberlist.Config.
Gossip gossip.Config `toml:"gossip"`
AntiEntropy struct {
Interval toml.Duration `toml:"interval"`
} `toml:"anti-entropy"`
Metric struct {
// Service can be statsd, expvar, or none.
Service string `toml:"service"`
// Host tells the statsd client where to write.
Host string `toml:"host"`
PollInterval toml.Duration `toml:"poll-interval"`
// Diagnostics toggles sending some limited diagnostic information to
// Pilosa's developers.
Diagnostics bool `toml:"diagnostics"`
} `toml:"metric"`
}
// NewConfig returns an instance of Config with default options.
func NewConfig() *Config {
c := &Config{
DataDir: "~/.pilosa",
Bind: ":10101",
MaxWritesPerRequest: 5000,
// LogPath: "",
// Verbose: false,
TLS: TLSConfig{},
}
// Cluster config.
c.Cluster.Disabled = false
// c.Cluster.Coordinator = false
c.Cluster.ReplicaN = 1
c.Cluster.Hosts = []string{}
c.Cluster.LongQueryTime = toml.Duration(time.Minute)
// Gossip config.
c.Gossip.Port = "14000"
// c.Gossip.Seeds = []string{}
// c.Gossip.Key = ""
c.Gossip.StreamTimeout = toml.Duration(10 * time.Second)
c.Gossip.SuspicionMult = 4
c.Gossip.PushPullInterval = toml.Duration(30 * time.Second)
c.Gossip.ProbeInterval = toml.Duration(1 * time.Second)
c.Gossip.ProbeTimeout = toml.Duration(500 * time.Millisecond)
c.Gossip.Interval = toml.Duration(200 * time.Millisecond)
c.Gossip.Nodes = 3
c.Gossip.ToTheDeadTime = toml.Duration(30 * time.Second)
// AntiEntropy config.
c.AntiEntropy.Interval = toml.Duration(10 * time.Minute)
// Metric config.
c.Metric.Service = "none"
// c.Metric.Host = ""
c.Metric.PollInterval = toml.Duration(0 * time.Minute)
c.Metric.Diagnostics = true
return c
}

19
config_test.go → server/config_test.go
View file

@ -12,34 +12,27 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
package server_test
import (
"reflect"
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/toml"
)
func Test_NewConfig(t *testing.T) {
c := pilosa.NewConfig()
c := server.NewConfig()
if c.Cluster.Disabled != pilosa.DefaultClusterDisabled {
if c.Cluster.Disabled {
t.Fatalf("unexpected Cluster.Disabled: %v", c.Cluster.Disabled)
}
// Ensure that hosts can't be specificed on a non-disabled cluster.
c.Cluster.Hosts = []string{c.Bind, "localhost:10102"}
// Change cluster type from the default (gossip) to an invalid string.
if err := c.Validate(); err != pilosa.ErrConfigClusterEnabledHosts {
t.Fatal(err)
}
}
func TestDuration(t *testing.T) {
d := pilosa.Duration(time.Second * 182)
d := toml.Duration(time.Second * 182)
if d.String() != "3m2s" {
t.Fatalf("Unexpected time Duration %s", d)
}

225
server/server.go
View file

@ -20,15 +20,18 @@
package server
import (
"errors"
"fmt"
"io"
"log"
"math/rand"
"net"
"net/http"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"crypto/tls"
@ -40,27 +43,24 @@ import (
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/statik"
"github.com/pilosa/pilosa/statsd"
"github.com/pkg/errors"
)
func init() {
rand.Seed(time.Now().UTC().UnixNano())
}
const (
// DefaultDataDir is the default data directory.
DefaultDataDir = "~/.pilosa"
)
type loggerLogger interface {
pilosa.Logger
Logger() *log.Logger
}
// Command represents the state of the pilosa server command.
type Command struct {
Server *pilosa.Server
// Configuration.
Config *pilosa.Config
// Profiling options.
CPUProfile string
CPUTime time.Duration
Config *Config
// Gossip transport
GossipTransport *gossip.Transport
@ -75,14 +75,13 @@ type Command struct {
// Passed to the Gossip implementation.
logOutput io.Writer
logger *log.Logger
logger loggerLogger
}
// NewCommand returns a new instance of Main.
func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command {
return &Command{
Server: pilosa.NewServer(),
Config: pilosa.NewConfig(),
Config: NewConfig(),
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
@ -91,8 +90,8 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command {
}
}
// Run executes the pilosa server.
func (m *Command) Run(args ...string) (err error) {
// Start starts the pilosa server - it returns once the server is running.
func (m *Command) Start() (err error) {
defer close(m.Started)
prefix := "~" + string(filepath.Separator)
if strings.HasPrefix(m.Config.DataDir, prefix) {
@ -120,81 +119,66 @@ func (m *Command) Run(args ...string) (err error) {
return fmt.Errorf("server.Open: %v", err)
}
m.Server.Logger.Printf("Listening as %s\n", m.Server.URI)
m.logger.Printf("Listening as %s\n", m.Server.URI)
return nil
}
// SetupLogger sets up the logger based on the configuration.
func (m *Command) SetupLogger() error {
// Wait waits for the server to be closed or interrupted.
func (m *Command) Wait() error {
// First SIGKILL causes server to shut down gracefully.
c := make(chan os.Signal, 2)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
select {
case sig := <-c:
m.logger.Printf("Received %s; gracefully shutting down...\n", sig.String())
// Second signal causes a hard shutdown.
go func() { <-c; os.Exit(1) }()
return errors.Wrap(m.Close(), "closing command")
case <-m.Done:
m.logger.Printf("Server closed externally")
return nil
}
}
// setupLogger sets up the logger based on the configuration.
func (m *Command) setupLogger() error {
var err error
if m.Config.LogPath == "" {
m.logOutput = m.Stderr
} else {
m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
return err
return errors.Wrap(err, "opening file")
}
}
if m.Config.Verbose {
vbl := pilosa.NewVerboseLogger(m.logOutput)
m.logger = vbl.Logger()
m.Server.Logger = vbl
m.logger = pilosa.NewVerboseLogger(m.logOutput)
} else {
sl := pilosa.NewStandardLogger(m.logOutput)
m.logger = sl.Logger()
m.Server.Logger = sl
m.logger = pilosa.NewStandardLogger(m.logOutput)
}
return nil
}
// SetupServer uses the cluster configuration to set up this server.
func (m *Command) SetupServer() error {
err := m.Config.Validate()
err := m.setupLogger()
if err != nil {
return err
return errors.Wrap(err, "setting up logger")
}
m.Server.Handler.Logger = m.Server.Logger
m.Server.Holder.Logger = m.Server.Logger
m.Server.Holder.Stats.SetLogger(m.Server.Logger)
handler := pilosa.NewHandler()
handler.Logger = m.logger
handler.FileSystem = &statik.FileSystem{}
handler.API = pilosa.NewAPI()
handler.API.Logger = m.logger
uri, err := pilosa.AddressWithDefaults(m.Config.Bind)
if err != nil {
return err
return errors.Wrap(err, "processing bind address")
}
m.Server.URI = *uri
cluster := pilosa.NewCluster()
cluster.ReplicaN = m.Config.Cluster.ReplicaN
cluster.Holder = m.Server.Holder
cluster.Logger = m.Server.Logger
m.Server.Cluster = cluster
// Configure data directory (for Cluster .topology)
m.Server.Cluster.Path = m.Config.DataDir
m.Server.NewAttrStore = boltdb.NewAttrStore
m.Server.Holder.NewAttrStore = boltdb.NewAttrStore
// Configure holder.
m.Server.Logger.Printf("Using data from: %s\n", m.Config.DataDir)
m.Server.Holder.Path = m.Config.DataDir
m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval)
if m.Config.Metric.Diagnostics {
m.Server.DiagnosticInterval = time.Duration(DefaultDiagnosticsInterval)
}
m.Server.SystemInfo = gopsutil.NewSystemInfo()
m.Server.GCNotifier = gcnotify.NewActiveGCNotifier()
m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
if err != nil {
return err
}
// Copy configuration flags.
m.Server.MaxWritesPerRequest = m.Config.MaxWritesPerRequest
// Setup TLS
var TLSConfig *tls.Config
@ -207,27 +191,73 @@ func (m *Command) SetupServer() error {
}
cert, err := tls.LoadX509KeyPair(m.Config.TLS.CertificatePath, m.Config.TLS.CertificateKeyPath)
if err != nil {
return err
return errors.Wrap(err, "load x509 key pair")
}
m.Server.TLS = &tls.Config{
TLSConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
InsecureSkipVerify: m.Config.TLS.SkipVerify,
}
TLSConfig = m.Server.TLS
}
c := pilosa.GetHTTPClient(TLSConfig)
m.Server.RemoteClient = c
m.Server.Handler.RemoteClient = c
m.Server.Cluster.RemoteClient = c
// Statik file system.
m.Server.Handler.FileSystem = &statik.FileSystem{}
diagnosticsInterval := time.Duration(0)
if m.Config.Metric.Diagnostics {
diagnosticsInterval = time.Duration(DefaultDiagnosticsInterval)
}
// Set configuration options.
m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval)
m.Server.Cluster.LongQueryTime = time.Duration(m.Config.Cluster.LongQueryTime)
return nil
statsClient, err := NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
if err != nil {
return errors.Wrap(err, "new stats client")
}
ln, err := getListener(*uri, TLSConfig)
if err != nil {
return errors.Wrap(err, "getting listener")
}
c := GetHTTPClient(TLSConfig)
handler.API.RemoteClient = c
m.Server, err = pilosa.NewServer(
pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)),
pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)),
pilosa.OptServerDataDir(m.Config.DataDir),
pilosa.OptServerReplicaN(m.Config.Cluster.ReplicaN),
pilosa.OptServerMaxWritesPerRequest(m.Config.MaxWritesPerRequest),
pilosa.OptServerMetricInterval(time.Duration(m.Config.Metric.PollInterval)),
pilosa.OptServerDiagnosticsInterval(diagnosticsInterval),
pilosa.OptServerLogger(m.logger),
pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore),
pilosa.OptServerHandler(handler),
pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),
pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
pilosa.OptServerStatsClient(statsClient),
pilosa.OptServerListener(ln),
pilosa.OptServerURI(uri),
pilosa.OptServerRemoteClient(c),
)
return errors.Wrap(err, "new server")
}
func GetHTTPClient(t *tls.Config) *http.Client {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
MaxIdleConns: 1000,
MaxIdleConnsPerHost: 200,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
if t != nil {
transport.TLSClientConfig = t
}
return &http.Client{Transport: transport}
}
// SetupNetworking sets up internode communication based on the configuration.
@ -255,13 +285,7 @@ func (m *Command) SetupNetworking() error {
return nil
}
// Set internal port (string).
gossipPortStr := pilosa.DefaultGossipPort
if m.Config.Gossip.Port != "" {
gossipPortStr = m.Config.Gossip.Port
}
gossipPort, err := strconv.Atoi(gossipPortStr)
gossipPort, err := strconv.Atoi(m.Config.Gossip.Port)
if err != nil {
return err
}
@ -272,7 +296,7 @@ func (m *Command) SetupNetworking() error {
if m.GossipTransport != nil {
transport = m.GossipTransport
} else {
transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger)
transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger())
if err != nil {
return err
}
@ -281,10 +305,12 @@ func (m *Command) SetupNetworking() error {
// Set Coordinator.
if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 {
m.Server.Cluster.Coordinator = m.Server.NodeID
m.Server.Cluster.Node.IsCoordinator = true
}
m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.Logger)
gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Config, m.Server, gossip.WithLogger(m.logger), gossip.WithTransport(transport))
gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger)
m.Server.Cluster.EventReceiver = gossipEventReceiver
gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, gossipEventReceiver, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport))
if err != nil {
return err
}
@ -318,7 +344,30 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) {
return pilosa.NewExpvarStatsClient(), nil
case "statsd":
return statsd.NewStatsClient(host)
default:
case "nop", "none":
return pilosa.NopStatsClient, nil
default:
return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].")
}
}
// getListener gets a net.Listener based on the config.
func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) {
// If bind URI has the https scheme, enable TLS
if uri.Scheme() == "https" && tlsconf != nil {
ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf)
if err != nil {
return nil, errors.Wrap(err, "tls.Listener")
}
} else if uri.Scheme() == "http" {
// Open HTTP listener to determine port (if specified as :0).
ln, err = net.Listen("tcp", uri.HostPort())
if err != nil {
return nil, errors.Wrap(err, "net.Listen")
}
} else {
return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme())
}
return ln, nil
}

61
server/server_test.go
View file

@ -26,9 +26,11 @@ import (
"strings"
"testing"
"testing/quick"
"time"
"github.com/BurntSushi/toml"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
@ -43,7 +45,7 @@ func TestMain_Set_Quick(t *testing.T) {
defer m.Close()
// Create client.
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil))
if err != nil {
t.Fatal(err)
}
@ -234,6 +236,49 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
}
// Ensure inverse slices get handled correctly in a multi-node query.
func TestMain_InverseSlices(t *testing.T) {
mains := test.MustRunMainWithCluster(t, 2)
m0 := mains[0]
m1 := mains[1]
// Make sure to use node0 in the cluster.
var m *test.Main
if m0.Server.NodeID < m1.Server.NodeID {
m = m0
} else {
m = m1
}
// Create frames.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create index:", err)
}
if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal("create frame:", err)
}
// Write data on cluster.
if _, err := m.Query("i", "", fmt.Sprintf(`
SetBit(col=1, frame="f", row=1000)
SetBit(col=1, frame="f", row=2000)
SetBit(col=1, frame="f", row=%d)
`, 1*pilosa.SliceWidth)); err != nil {
t.Fatal("setting bits:", err)
}
time.Sleep(1 * time.Second)
// Query the cluster.
if res, err := m.Query("i", "", `Bitmap(col=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != fmt.Sprintf(`{"results":[{"attrs":{},"bits":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" {
t.Fatalf("unexpected result: %s", res)
}
}
// Ensure program can set bits on one cluster and then restore to a second cluster.
func TestMain_FrameRestore(t *testing.T) {
mains1 := test.MustRunMainWithCluster(t, 2)
@ -277,11 +322,11 @@ func TestMain_FrameRestore(t *testing.T) {
defer m21.Close()
// Import from first cluster.
client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), server.GetHTTPClient(nil))
if err != nil {
t.Fatal("new client:", err)
}
client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), server.GetHTTPClient(nil))
if err != nil {
t.Fatal("new client:", err)
}
@ -375,7 +420,11 @@ func TestMain_RecalculateHashes(t *testing.T) {
}
// Calculate caches on the first node
cluster[0].RecalculateCaches()
err := cluster[0].RecalculateCaches()
if err != nil {
t.Fatalf("recalculating caches: %v", err)
}
target := `{"results":[[{"id":7,"count":99},{"id":1,"count":99},{"id":9,"count":99},{"id":5,"count":99},{"id":4,"count":99},{"id":8,"count":99},{"id":2,"count":99},{"id":6,"count":99},{"id":3,"count":99}]]}`
// Run a TopN query on all nodes. The result should be the same as the target.
@ -441,8 +490,8 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand {
}
// ParseConfig parses s into a Config.
func ParseConfig(s string) (pilosa.Config, error) {
var c pilosa.Config
func ParseConfig(s string) (server.Config, error) {
var c server.Config
_, err := toml.Decode(s, &c)
return c, err
}

35
server_test.go Normal file
View file

@ -0,0 +1,35 @@
package pilosa_test
import (
"context"
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
// TestMonitorAntiEntropy is a regression test which which caught a bug where
// pilosa.Server was not having its remoteClient field set by an option and so
// it was using a nil client in monitorAntiEntropy.
func TestMonitorAntiEntropy(t *testing.T) {
cluster := test.MustRunMainWithCluster(t, 3, test.OptAntiEntropyInterval(time.Millisecond*1))
client := cluster[1].Client()
err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
err = client.CreateFrame(context.Background(), "balh", "fralh", pilosa.FrameOptions{})
if err != nil {
t.Fatalf("creating frame: %v", err)
}
time.Sleep(time.Millisecond * 2)
for _, m := range cluster {
err := m.Close()
if err != nil {
t.Fatal(err)
}
}
}

16
stats_test.go
View file

@ -215,10 +215,10 @@ func TestStatsCount_CreateIndex(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
called := false
s.Handler.Holder.Stats = &MockStats{
s.Handler.API.Holder.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "createIndex" {
t.Errorf("Expected createIndex, Results %s", name)
@ -239,7 +239,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Create index.
@ -247,7 +247,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) {
t.Fatal(err)
}
called := false
s.Handler.Holder.Stats = &MockStats{
s.Handler.API.Holder.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "deleteIndex" {
t.Errorf("Expected deleteIndex, Results %s", name)
@ -268,7 +268,7 @@ func TestStatsCount_CreateFrame(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Create index.
@ -276,7 +276,7 @@ func TestStatsCount_CreateFrame(t *testing.T) {
t.Fatal(err)
}
called := false
s.Handler.Holder.Stats = &MockStats{
s.Handler.API.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "createFrame" {
t.Errorf("Expected createFrame, Results %s", name)
@ -300,7 +300,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
called := false
// Create index.
@ -308,7 +308,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) {
if _, err := indx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
s.Handler.Holder.Stats = &MockStats{
s.Handler.API.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "deleteFrame" {
t.Errorf("Expected deleteFrame, Results %s", name)

5
test/cluster.go
View file

@ -50,6 +50,7 @@ func NewCluster(n int) *pilosa.Cluster {
c.Node = c.Nodes[0]
c.Coordinator = c.Nodes[0].ID
c.SetState(pilosa.ClusterStateNormal)
return c
}
@ -129,7 +130,7 @@ func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / pilosa.SliceWidth
nodes := c0.FragmentNodes(index, slice)
nodes := c0.SliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
@ -153,7 +154,7 @@ func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name s
// Determine which node should receive the SetFieldValue.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := columnID / pilosa.SliceWidth
nodes := c0.FragmentNodes(index, slice)
nodes := c0.SliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)

3
test/executor.go
View file

@ -20,6 +20,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/server"
)
// Executor represents a test wrapper for pilosa.Executor.
@ -30,7 +31,7 @@ type Executor struct {
var remoteClient *http.Client
func init() {
remoteClient = pilosa.GetHTTPClient(nil)
remoteClient = server.GetHTTPClient(nil)
}
// NewExecutor returns a new instance of Executor.

17
test/handler.go
View file

@ -40,12 +40,12 @@ func NewHandler() *Handler {
h := &Handler{
Handler: pilosa.NewHandler(),
}
h.Handler.Executor = &h.Executor
h.API = pilosa.NewAPI()
h.Handler.API = h.API
h.Handler.API.Executor = &h.Executor
// Handler test messages can no-op.
h.Broadcaster = pilosa.NopBroadcaster
h.SetNormal()
h.API.Broadcaster = pilosa.NopBroadcaster
return h
}
@ -80,14 +80,13 @@ func NewServer() *Server {
if err != nil {
panic(err)
}
s.Handler.API.URI = *uri
// Handler test messages can no-op.
s.Handler.Broadcaster = pilosa.NopBroadcaster
s.Handler.API.Broadcaster = pilosa.NopBroadcaster
// Create a default cluster on the handler
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster.Nodes[0].URI = *uri
s.Handler.Node = s.Handler.Cluster.Nodes[0]
s.Handler.API.Cluster = NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
return s
}

62
test/pilosa.go
View file

@ -23,11 +23,13 @@ import (
"os"
"strings"
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/toml"
"github.com/pkg/errors"
)
@ -41,23 +43,40 @@ type Main struct {
Stderr bytes.Buffer
}
type MainOpt func(m *Main) error
func OptAntiEntropyInterval(dur time.Duration) MainOpt {
return func(m *Main) error {
m.Command.Config.AntiEntropy.Interval = toml.Duration(dur)
return nil
}
}
// NewMain returns a new instance of Main with a temporary data directory and random port.
func NewMain() *Main {
func NewMain(opts ...MainOpt) *Main {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)}
m.Server.Network = *Network
m.Server.NewAttrStore = NewAttrStore
m.Server.Holder.NewAttrStore = NewAttrStore
m.Config.DataDir = path
m.Config.Bind = "http://localhost:0"
m.Config.Cluster.Disabled = true
m.Command.Stdin = &m.Stdin
m.Command.Stdout = &m.Stdout
m.Command.Stderr = &m.Stderr
for _, opt := range opts {
err := opt(m)
if err != nil {
panic(err)
}
}
err = m.SetupServer()
if err != nil {
panic(err)
}
if testing.Verbose() {
m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
@ -68,8 +87,8 @@ func NewMain() *Main {
}
// NewMainWithCluster returns a new instance of Main with clustering enabled.
func NewMainWithCluster(isCoordinator bool) *Main {
m := NewMain()
func NewMainWithCluster(isCoordinator bool, opts ...MainOpt) *Main {
m := NewMain(opts...)
m.Config.Cluster.Disabled = false
m.Config.Cluster.Coordinator = isCoordinator
return m
@ -77,8 +96,8 @@ func NewMainWithCluster(isCoordinator bool) *Main {
// MustRunMainWithCluster ruturns a running array of *Main where
// all nodes are joined via memberlist (i.e. clustering enabled).
func MustRunMainWithCluster(t *testing.T, size int) []*Main {
ma, err := runMainWithCluster(size)
func MustRunMainWithCluster(t *testing.T, size int, opts ...MainOpt) []*Main {
ma, err := runMainWithCluster(size, opts...)
if err != nil {
t.Fatalf("new main array with cluster: %v", err)
}
@ -87,7 +106,7 @@ func MustRunMainWithCluster(t *testing.T, size int) []*Main {
// runMainWithCluster runs an array of *Main where all nodes are
// joined via memberlist (i.e. clustering enabled).
func runMainWithCluster(size int) ([]*Main, error) {
func runMainWithCluster(size int, opts ...MainOpt) ([]*Main, error) {
if size == 0 {
return nil, errors.New("cluster must contain at least one node")
}
@ -100,7 +119,8 @@ func runMainWithCluster(size int) ([]*Main, error) {
var gossipSeeds = make([]string, size)
for i := 0; i < size; i++ {
m := NewMainWithCluster(i == 0)
m := NewMainWithCluster(i == 0, opts...)
m.Config.Cluster.Disabled = false
gossipSeeds[i], err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i])
if err != nil {
@ -117,7 +137,7 @@ func runMainWithCluster(size int) ([]*Main, error) {
func MustRunMain() *Main {
m := NewMain()
m.Config.Metric.Diagnostics = false // Disable diagnostics.
if err := m.Run(); err != nil {
if err := m.Start(); err != nil {
panic(err)
}
return m
@ -136,15 +156,19 @@ func (m *Main) Reopen() error {
}
// Create new main with the same config.
config := m.Config
config := m.Command.Config
m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr)
m.Server.Network = *Network
m.Command.Config = config
err := m.SetupServer()
if err != nil {
return errors.Wrap(err, "setting up server")
}
m.Server.NewAttrStore = boltdb.NewAttrStore
m.Server.Holder.NewAttrStore = m.Server.NewAttrStore
m.Config = config
// Run new program.
if err := m.Run(); err != nil {
if err := m.Start(); err != nil {
return err
}
return nil
@ -174,12 +198,6 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) (
return seed, err
}
// Open server listener.
err = m.Server.OpenListener()
if err != nil {
return seed, err
}
// Open gossip transport to use in SetupServer.
transport, err := gossip.NewTransport(host, bindPort, nil)
if err != nil {
@ -221,7 +239,7 @@ func (m *Main) URL() string { return "http://" + m.Server.Addr().String() }
// Client returns a client to connect to the program.
func (m *Main) Client() *pilosa.InternalHTTPClient {
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil))
if err != nil {
panic(err)
}

7
test/test.go
View file

@ -13,10 +13,3 @@
// limitations under the License.
package test
import "flag"
// Test flags.
var (
Network = flag.String("network", "tcp", "network name")
)

30
toml/toml.go Normal file
View file

@ -0,0 +1,30 @@
package toml
import "time"
// Duration is a TOML wrapper type for time.Duration.
type Duration time.Duration
// String returns the string representation of the duration.
func (d Duration) String() string { return time.Duration(d).String() }
// UnmarshalText parses a TOML value into a duration value.
func (d *Duration) UnmarshalText(text []byte) error {
v, err := time.ParseDuration(string(text))
if err != nil {
return err
}
*d = Duration(v)
return nil
}
// MarshalText writes duration value in text format.
func (d Duration) MarshalText() (text []byte, err error) {
return []byte(d.String()), nil
}
// MarshalTOML write duration into valid TOML.
func (d Duration) MarshalTOML() ([]byte, error) {
return []byte(d.String()), nil
}

425
utils_test.go Normal file
View file

@ -0,0 +1,425 @@
package pilosa
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"path/filepath"
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
)
// NewTestCluster returns a cluster with n nodes and uses a mod-based hasher.
func NewTestCluster(n int) *Cluster {
path, err := ioutil.TempDir("", "pilosa-cluster-")
if err != nil {
panic(err)
}
c := NewCluster()
c.ReplicaN = 1
c.Hasher = NewTestModHasher()
c.Path = path
c.Topology = NewTopology()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &Node{
ID: fmt.Sprintf("node%d", i),
URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)),
})
}
c.Node = c.Nodes[0]
c.Coordinator = c.Nodes[0].ID
c.SetState(ClusterStateNormal)
return c
}
// NewTestURI is a test URI creator that intentionally swallows errors.
func NewTestURI(scheme, host string, port uint16) URI {
uri := DefaultURI()
uri.SetScheme(scheme)
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
func NewTestURIFromHostPort(host string, port uint16) URI {
uri := DefaultURI()
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
// ModHasher represents a simple, mod-based hashing.
type TestModHasher struct{}
// NewTestModHasher returns a new instance of ModHasher with n buckets.
func NewTestModHasher() *TestModHasher { return &TestModHasher{} }
func (*TestModHasher) Hash(key uint64, n int) int { return int(key) % n }
// ClusterCluster represents a cluster of test nodes, each of which
// has a Cluster.
// ClusterCluster implements Broadcaster interface.
type ClusterCluster struct {
Clusters []*Cluster
common *commonClusterSettings
mu sync.RWMutex
resizing bool
resizeDone chan struct{}
}
type commonClusterSettings struct {
Nodes []*Node
}
func (t *ClusterCluster) CreateIndex(name string) error {
for _, c := range t.Clusters {
if _, err := c.Holder.CreateIndexIfNotExists(name, IndexOptions{}); err != nil {
return err
}
}
return nil
}
func (t *ClusterCluster) CreateFrame(index, frame string, opt FrameOptions) error {
for _, c := range t.Clusters {
idx, err := c.Holder.CreateIndexIfNotExists(index, IndexOptions{})
if err != nil {
return err
}
if _, err := idx.CreateFrame(frame, opt); err != nil {
return err
}
}
return nil
}
func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error {
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / SliceWidth
nodes := c0.SliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
}
_, err := f.SetBit(view, rowID, colID, x)
if err != nil {
return err
}
}
return nil
}
func (t *ClusterCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error {
// Determine which node should receive the SetFieldValue.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := columnID / SliceWidth
nodes := c0.SliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
}
_, err := f.SetFieldValue(columnID, name, value)
if err != nil {
return err
}
}
return nil
}
func (t *ClusterCluster) clusterByID(id string) *Cluster {
for _, c := range t.Clusters {
if c.Node.ID == id {
return c
}
}
return nil
}
// AddNode adds a node to the cluster and (potentially) starts a resize job.
func (t *ClusterCluster) AddNode(saveTopology bool) error {
id := len(t.Clusters)
c, err := t.addCluster(id, saveTopology)
if err != nil {
return err
}
// Send NodeJoin event to coordinator.
if id > 0 {
coord := t.Clusters[0]
ev := &NodeEvent{
Event: NodeJoin,
Node: c.Node,
}
if err := coord.ReceiveEvent(ev); err != nil {
return err
}
// Wait for the AddNode job to finish.
if c.State() != ClusterStateNormal {
t.resizeDone = make(chan struct{})
t.mu.Lock()
t.resizing = true
t.mu.Unlock()
<-t.resizeDone
}
}
return nil
}
// WriteTopology writes the given topology to disk.
func (t *ClusterCluster) WriteTopology(path string, top *Topology) error {
if buf, err := proto.Marshal(top.Encode()); err != nil {
return err
} else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil {
return err
}
return nil
}
func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*Cluster, error) {
id := fmt.Sprintf("node%d", i)
uri := NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0))
node := &Node{
ID: id,
URI: uri,
}
// add URI to common
//t.common.NodeIDs = append(t.common.NodeIDs, id)
//sort.Sort(t.common.NodeIDs)
// add node to common
t.common.Nodes = append(t.common.Nodes, node)
// create node-specific temp directory
path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i))
if err != nil {
return nil, err
}
// holder
h := NewHolder()
h.Path = path
// cluster
c := NewCluster()
c.ReplicaN = 1
c.Hasher = NewTestModHasher()
c.Path = path
c.Topology = NewTopology()
c.Holder = h
c.MemberSet = NewStaticMemberSet(c.Nodes)
c.Node = node
c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator
c.Broadcaster = t
// add nodes
if saveTopology {
for _, n := range t.common.Nodes {
c.AddNode(n)
}
}
// Add this node to the ClusterCluster.
t.Clusters = append(t.Clusters, c)
return c, nil
}
// NewClusterCluster returns a new instance of test.Cluster.
func NewClusterCluster(n int) *ClusterCluster {
tc := &ClusterCluster{
common: &commonClusterSettings{},
}
// add clusters
for i := 0; i < n; i++ {
_, err := tc.addCluster(i, true)
if err != nil {
panic(err)
}
}
return tc
}
// SetState sets the state of the cluster on each node.
func (t *ClusterCluster) SetState(state string) {
for _, c := range t.Clusters {
c.SetState(state)
}
}
// Open opens all clusters in the test cluster.
func (t *ClusterCluster) Open() error {
for _, c := range t.Clusters {
if err := c.Open(); err != nil {
return err
}
if err := c.Holder.Open(); err != nil {
return err
}
if err := c.SetNodeState(NodeStateReady); err != nil {
return err
}
}
// Start the listener on the coordinator.
if len(t.Clusters) == 0 {
return nil
}
t.Clusters[0].ListenForJoins()
return nil
}
// Close closes all clusters in the test cluster.
func (t *ClusterCluster) Close() error {
for _, c := range t.Clusters {
err := c.Close()
if err != nil {
return err
}
}
return nil
}
// SendSync is a test implemenetation of Broadcaster SendSync method.
func (t *ClusterCluster) SendSync(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ClusterStatus:
// Apply the send message to all nodes (except the coordinator).
for _, c := range t.Clusters {
c.MergeClusterStatus(obj)
}
t.mu.RLock()
if obj.State == ClusterStateNormal && t.resizing {
close(t.resizeDone)
}
t.mu.RUnlock()
}
return nil
}
// SendAsync is a test implemenetation of Broadcaster SendAsync method.
func (t *ClusterCluster) SendAsync(pb proto.Message) error {
return nil
}
// SendTo is a test implemenetation of Broadcaster SendTo method.
func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ResizeInstruction:
err := t.FollowResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
coord := t.clusterByID(to.ID)
go coord.MarkResizeInstructionComplete(obj)
}
return nil
}
// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
// Prepare the return message.
complete := &internal.ResizeInstructionComplete{
JobID: instr.JobID,
Node: instr.Node,
Error: "",
}
// Stop processing on any error.
if err := func() error {
// figure out which node it was meant for, then call the operation on that cluster
// basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI)
instrNode := DecodeNode(instr.Node)
destCluster := t.clusterByID(instrNode.ID)
// Sync the schema received in the resize instruction.
if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil {
return err
}
for _, src := range instr.Sources {
srcNode := DecodeNode(src.Node)
srcCluster := t.clusterByID(srcNode.ID)
srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
if destFragment == nil {
// Create fragment on destination if it doesn't exist.
f := destCluster.Holder.Frame(src.Index, src.Frame)
v := f.View(src.View)
var err error
destFragment, err = v.CreateFragmentIfNotExists(src.Slice)
if err != nil {
return err
}
}
buf := bytes.NewBuffer(nil)
bw := bufio.NewWriter(buf)
br := bufio.NewReader(buf)
// Get the fragment from source.
if _, err := srcFragment.WriteTo(bw); err != nil {
return err
}
// Flush the bufio.buf to the io.Writer (buf).
bw.Flush()
// Write data to destination.
if _, err := destFragment.ReadFrom(br); err != nil {
return err
}
}
return nil
}(); err != nil {
complete.Error = err.Error()
}
node := DecodeNode(instr.Coordinator)
if err := t.SendTo(node, complete); err != nil {
return err
}
return nil
}

43
view.go
View file

@ -357,6 +357,49 @@ func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err e
return sum, count, nil
}
// FieldMin returns the min and count of a field.
func (v *View) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) {
var minHasValue bool
for _, f := range v.Fragments() {
fmin, fcount, err := f.FieldMin(filter, bitDepth)
if err != nil {
return min, count, err
}
// Don't consider a min based on zero columns.
if fcount == 0 {
continue
}
if !minHasValue {
min = fmin
minHasValue = true
count += fcount
continue
}
if fmin < min {
min = fmin
count += fcount
}
}
return min, count, nil
}
// FieldMax returns the max and count of a field.
func (v *View) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) {
for _, f := range v.Fragments() {
fmax, fcount, err := f.FieldMax(filter, bitDepth)
if err != nil {
return max, count, err
}
if fcount > 0 && fmax > max {
max = fmax
count += fcount
}
}
return max, count, nil
}
// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) {
bm := NewBitmap()