diff --git a/.travis.yml b/.travis.yml index 2377d30f1..6a98fe521 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,9 @@ deploy: skip_cleanup: true on: branch: master +matrix: + allow_failures: + - go: master notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" diff --git a/CHANGELOG.md b/CHANGELOG.md index cb7141d75..0a54a5b73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,51 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [0.8.0] - 2017-11-15 + +This version contains 31 contributions from 8 contributors. There are 84 files changed, 3,732 insertions, and 1,428 deletions. + +### Added + +- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895)) +- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933)) +- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938)) +- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942)) +- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867)) +- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898)) +- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889)) +- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881)) +- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955)) + +### Changed + +- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943)) +- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884)) +- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892)) +- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945)) +- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) + +### Fixed + +- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941)) +- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934)) +- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919)) +- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887)) +- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882)) +- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949)) + +### Performance + +- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874)) + +## [0.7.2] - 2017-11-15 + +This version contains 1 contribution from 1 contributor. There is 1 file changed, 16 insertions, and 1 deletion. + +### Changed + +- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920)) + ## [0.7.1] - 2017-10-09 This version contains 3 contributions from 3 contributors. There are 14 files changed, 221 insertions, and 52 deletions. @@ -163,3 +208,6 @@ This version contains 53 contributions from 13 contributors (including 4 volunte [Unreleased]: https://github.com/pilosa/pilosa/compare/v0.5...HEAD [0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4 [0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5 +[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6 +[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7 +[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8 diff --git a/Dockerfile b/Dockerfile index f129ce32c..b145465b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.9.1 as builder +FROM golang:1.9.2 as builder ARG ldflags='' @@ -18,4 +18,4 @@ EXPOSE 10101 VOLUME /data ENTRYPOINT ["/pilosa"] -CMD ["server", "--data-dir", "/data"] +CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"] diff --git a/Gopkg.lock b/Gopkg.lock index 4b6cf23ac..335482e9d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -4,7 +4,8 @@ [[projects]] name = "github.com/BurntSushi/toml" packages = ["."] - revision = "99064174e013895bbd9b025c31100bd1d9b590ca" + revision = "b26d9c308763d68093482582cea63d69be07a0f0" + version = "v0.3.0" [[projects]] branch = "master" @@ -52,13 +53,13 @@ branch = "master" name = "github.com/golang/groupcache" packages = ["lru"] - revision = "b710c8433bd175204919eb38776e944233235d03" + revision = "84a468cf14b4376def5d68c722b139b881c450a4" [[projects]] branch = "master" name = "github.com/golang/protobuf" packages = ["proto"] - revision = "130e6b02ab059e7b717a096f397c5b60111cae74" + revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9" [[projects]] name = "github.com/gorilla/context" @@ -69,8 +70,8 @@ [[projects]] name = "github.com/gorilla/mux" packages = ["."] - revision = "24fca303ac6da784b9e8269f724ddeb0b2eea5e7" - version = "v1.5.0" + revision = "7f08801859139f86dfafd1c296e2cba9a80d292e" + version = "v1.6.0" [[projects]] branch = "master" @@ -100,7 +101,7 @@ branch = "master" name = "github.com/hashicorp/go-sockaddr" packages = ["."] - revision = "41949a141473f6340abc6ba0fcd0f89da6f6f837" + revision = "9b4c5fa5b10a683339a270d664474b9f4aee62fc" [[projects]] branch = "master" @@ -112,7 +113,7 @@ branch = "master" name = "github.com/hashicorp/hcl" packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] - revision = "68e816d1c783414e79bc65b3994d9ab6b0a722ab" + revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" [[projects]] name = "github.com/hashicorp/memberlist" @@ -136,13 +137,13 @@ branch = "master" name = "github.com/miekg/dns" packages = [".","internal/socket"] - revision = "946bd9fbed05568b0f3cd188353d8aa28f38b688" + revision = "9fc4eb252eedf0ef8adc05169ce35da5e31beaba" [[projects]] branch = "master" name = "github.com/mitchellh/mapstructure" packages = ["."] - revision = "d0303fe809921458f417bcf828397a65db30a7e4" + revision = "06020f85339e21b2478f756a78e295255ffa4d6a" [[projects]] name = "github.com/pelletier/go-toml" @@ -172,7 +173,7 @@ branch = "master" name = "github.com/spf13/afero" packages = [".","mem"] - revision = "e67d870304c4bca21331b02f414f970df13aa694" + revision = "5660eeed305fe5f69c8fc6cf899132a459a97064" [[projects]] name = "github.com/spf13/cast" @@ -181,10 +182,10 @@ version = "v1.1.0" [[projects]] - branch = "master" name = "github.com/spf13/cobra" packages = ["."] - revision = "50204810fdb5010baae72e4f41b303689cbdcc9f" + revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" + version = "v0.0.1" [[projects]] branch = "master" @@ -199,34 +200,34 @@ version = "v1.0.0" [[projects]] - branch = "master" name = "github.com/spf13/viper" packages = ["."] - revision = "d9cca5ef33035202efb1586825bdbb15ff9ec3ba" + revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" + version = "v1.0.0" [[projects]] branch = "master" name = "golang.org/x/net" packages = ["context"] - revision = "a04bdaca5b32abe1c069418fb7088ae607de5bd0" + revision = "a337091b0525af65de94df2eb7e98bd9962dcbe2" [[projects]] branch = "master" name = "golang.org/x/sync" packages = ["errgroup"] - revision = "8e0aa688b654ef28caa72506fa5ec8dba9fc7690" + revision = "fd80eb99c8f653c847d294a001bdf2a3a6f768f5" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "ebfc5b4631820b793c9010c87fd8fef0f39eb082" + revision = "1e2299c37cc91a509f1b12369872d27be0ce98a6" [[projects]] branch = "master" name = "golang.org/x/text" packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"] - revision = "825fc78a2fd6fa0a5447e300189e3219e05e1f25" + revision = "88f656faf3f37f690df1a32515b479415e1a6769" [[projects]] branch = "v2" @@ -237,6 +238,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "4dd559e1f44fd2ad3032f1076f627d86736f47eac961527cddbef21b0c7c0006" + inputs-digest = "2a604b59afbfef0e6d4ad5e2eaf63f5fff19e767234e44974cf7aead0510e0fb" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index a91539345..7ffa67a2f 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,49 +1,3 @@ - -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - -[[constraint]] - name = "github.com/BurntSushi/toml" - revision = "99064174e013895bbd9b025c31100bd1d9b590ca" - -[[constraint]] - branch = "master" - name = "github.com/CAFxX/gcnotifier" - -[[constraint]] - name = "github.com/boltdb/bolt" - version = "1.3.0" - -[[constraint]] - name = "github.com/gorilla/mux" - version = "1.3.0" - -[[constraint]] - name = "github.com/hashicorp/memberlist" - version = "=0.1.0" - -[[constraint]] - branch = "master" - name = "github.com/spf13/cobra" - -[[constraint]] - branch = "master" - name = "github.com/spf13/viper" +# This file intentionally left blank as all needed dependencies are imported by +# the project and thus tracked by `dep`. +# See https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md for details. diff --git a/Makefile b/Makefile index 33f1c579d..9987810d3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz +.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz clean docker-build docker-test DEP := $(shell command -v dep 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) @@ -9,9 +9,13 @@ CLONE_URL=github.com/pilosa/pilosa PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) BUILD_TIME=`date -u +%FT%T%z` LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" +DOCKER_GOLANG_IMAGE=golang:latest default: test pilosa +clean: + rm -rf vendor build + $(GOPATH)/bin: mkdir $(GOPATH)/bin @@ -51,15 +55,19 @@ pilosa: vendor go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa release-build: vendor +ifdef DOCKER_BUILD + make docker-build FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" +else make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" +endif cp LICENSE README.md build/pilosa-$(IDENTIFIER) tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/ @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz" release: - make release-build GOOS=linux GOARCH=amd64 - make release-build GOOS=linux GOARCH=386 make release-build GOOS=darwin GOARCH=amd64 + make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 + make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 prerelease-build: vendor make pilosa FLAGS="-o build/pilosa-master-$(GOOS)-$(GOARCH)/pilosa" @@ -99,3 +107,9 @@ endif docker: docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) . @echo "Created image: pilosa:$(VERSION)" + +docker-build: + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) $(DOCKER_GOLANG_IMAGE) go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + +docker-test: + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) $(DOCKER_GOLANG_IMAGE) go test $(TESTFLAGS) $(PKGS) diff --git a/cache.go b/cache.go index 2e66d35b9..a6f37a409 100644 --- a/cache.go +++ b/cache.go @@ -215,7 +215,7 @@ func (c *RankCache) IDs() []uint64 { return a } -// Invalidate recalculates the the entries by rank. +// Invalidate recalculates the entries by rank. func (c *RankCache) Invalidate() { c.mu.Lock() defer c.mu.Unlock() diff --git a/client.go b/client.go index 04a9af9e8..1515892c7 100644 --- a/client.go +++ b/client.go @@ -45,14 +45,13 @@ type ClientOptions struct { // InternalHTTPClient represents a client to the Pilosa cluster. type InternalHTTPClient struct { defaultURI *URI - options *ClientOptions // The client to use for HTTP communication. HTTPClient *http.Client } // NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host. -func NewInternalHTTPClient(host string, options *ClientOptions) (*InternalHTTPClient, error) { +func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTTPClient, error) { if host == "" { return nil, ErrHostRequired } @@ -62,22 +61,14 @@ func NewInternalHTTPClient(host string, options *ClientOptions) (*InternalHTTPCl return nil, err } - client := NewInternalHTTPClientFromURI(uri, options) + client := NewInternalHTTPClientFromURI(uri, remoteClient) return client, nil } -func NewInternalHTTPClientFromURI(defaultURI *URI, options *ClientOptions) *InternalHTTPClient { - if options == nil { - options = &ClientOptions{} - } - transport := &http.Transport{} - if options.TLS != nil { - transport.TLSClientConfig = options.TLS - } - client := &http.Client{Transport: transport} +func NewInternalHTTPClientFromURI(defaultURI *URI, remoteClient *http.Client) *InternalHTTPClient { return &InternalHTTPClient{ defaultURI: defaultURI, - HTTPClient: client, + HTTPClient: remoteClient, } } diff --git a/client_test.go b/client_test.go index d81e76fa8..4ba603176 100644 --- a/client_test.go +++ b/client_test.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "net/http" "reflect" "testing" @@ -43,6 +44,13 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { return server, hldr } +var defaultClient *http.Client + +func init() { + defaultClient = pilosa.GetHTTPClient(nil) + +} + // Test distributed TopN Row count across 3 nodes. func TestClient_MultiNode(t *testing.T) { cluster := test.NewCluster(3) @@ -54,21 +62,21 @@ func TestClient_MultiNode(t *testing.T) { } s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[0].Holder e.URI = cluster.Nodes[0].URI e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[1].Holder e.URI = cluster.Nodes[1].URI e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[2].Holder e.URI = cluster.Nodes[2].URI e.Cluster = cluster @@ -132,9 +140,9 @@ func TestClient_MultiNode(t *testing.T) { // Connect to each node to compare results. client := make([]*test.Client, 3) - client[0] = test.MustNewClient(s[0].Host()) - client[1] = test.MustNewClient(s[1].Host()) - client[2] = test.MustNewClient(s[2].Host()) + client[0] = test.MustNewClient(s[0].Host(), defaultClient) + client[1] = test.MustNewClient(s[1].Host(), defaultClient) + client[2] = test.MustNewClient(s[2].Host(), defaultClient) topN := 4 queryRequest := &internal.QueryRequest{ @@ -215,7 +223,7 @@ func TestClient_Import(t *testing.T) { s.Handler.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, @@ -266,7 +274,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) { s.Handler.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, @@ -315,7 +323,7 @@ func TestClient_ImportValue(t *testing.T) { s.Handler.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{ {ColumnID: 1, Value: -10}, {ColumnID: 2, Value: 20}, @@ -352,7 +360,7 @@ func TestClient_BackupRestore(t *testing.T) { s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) // Backup from frame. var buf bytes.Buffer @@ -417,7 +425,7 @@ func TestClient_BackupInverseView(t *testing.T) { s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) // Backup from frame. var buf bytes.Buffer @@ -454,7 +462,7 @@ func TestClient_BackupInvalidView(t *testing.T) { s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) // Backup from frame. var buf bytes.Buffer @@ -484,7 +492,7 @@ func TestClient_FragmentBlocks(t *testing.T) { s.Handler.Holder = hldr.Holder // Retrieve blocks. - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) blocks, err := c.FragmentBlocks(context.Background(), "i", "f", pilosa.ViewStandard, 0) if err != nil { t.Fatal(err) diff --git a/cmd/server_test.go b/cmd/server_test.go index b09962663..b650e316a 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -45,10 +45,11 @@ func TestServerConfig(t *testing.T) { // TEST 0 { args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"}, - env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_POLL_INTERVAL": "3m2s"}, + env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_POLL_INTERVAL": "3m2s", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, cfgFileContent: ` data-dir = "/tmp/myFileDatadir" bind = "localhost:0" + max-writes-per-request = 3000 [cluster] type = "static" @@ -56,6 +57,7 @@ func TestServerConfig(t *testing.T) { hosts = [ "localhost:19444", ] + long-query-time = "1m10s" `, validation: func() error { v := validator{} @@ -63,6 +65,9 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Bind, "localhost:10111") v.Check(cmd.Server.Config.Cluster.ReplicaN, 2) v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"}) + v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Second*182)) + v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90)) + v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) return v.Error() }, }, @@ -78,13 +83,10 @@ func TestServerConfig(t *testing.T) { hosts = [ "localhost:19444", ] - [plugins] - path = "/var/sloth" `, validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"}) - v.Check(cmd.Server.Config.Plugins.Path, "/var/sloth") v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9)) return v.Error() }, diff --git a/config.go b/config.go index e6138802e..1b4a4485a 100644 --- a/config.go +++ b/config.go @@ -78,13 +78,10 @@ type Config struct { ReplicaN int `toml:"replicas"` Type string `toml:"type"` Hosts []string `toml:"hosts"` + PollInterval Duration `toml:"poll-interval"` LongQueryTime Duration `toml:"long-query-time"` } `toml:"cluster"` - Plugins struct { - Path string `toml:"path"` - } `toml:"plugins"` - AntiEntropy struct { Interval Duration `toml:"interval"` } `toml:"anti-entropy"` diff --git a/ctl/common.go b/ctl/common.go index dc64c0bee..826462b8c 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -2,6 +2,7 @@ package ctl import ( "crypto/tls" + "github.com/pilosa/pilosa" "github.com/spf13/pflag" ) @@ -22,19 +23,18 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP // CommandClient returns a pilosa.InternalHTTPClient for the command func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error) { tlsConfig := cmd.TLSConfiguration() - var clientOptions *pilosa.ClientOptions + var TLSConfig *tls.Config if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" { cert, err := tls.LoadX509KeyPair(tlsConfig.CertificatePath, tlsConfig.CertificateKeyPath) if err != nil { return nil, err } - TLSConfig := &tls.Config{ + TLSConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: tlsConfig.SkipVerify, } - clientOptions = &pilosa.ClientOptions{TLS: TLSConfig} } - client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), clientOptions) + client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), pilosa.GetHTTPClient(TLSConfig)) if err != nil { return nil, err } diff --git a/ctl/generate_config.go b/ctl/generate_config.go index 151076ca8..a64678475 100644 --- a/ctl/generate_config.go +++ b/ctl/generate_config.go @@ -59,9 +59,6 @@ max-writes-per-request = 5000 service = "statsd" host = "127.0.0.1:8125" poll-interval = "0m15s" - -[plugins] - path = "" `)+"\n") return nil } diff --git a/ctl/server.go b/ctl/server.go index 51043a483..07837caa0 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -35,8 +35,8 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") - flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Long Query Time.") - flags.StringVarP(&srv.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") + flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? + flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path") flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index c653a7052..b1453db22 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -172,10 +172,10 @@ func (d *Diagnostics) CompareVersion(value string) error { if localVersion[0] < currentVersion[0] { //Major return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value) - } else if localVersion[1] < currentVersion[1] { // Minor + } else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value) - } else if localVersion[2] < currentVersion[2] { // Patch - return fmt.Errorf("There is a new patch release of Pilosa availbale: %s: https://github.com/pilosa/pilosa/releases", value) + } else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch + return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value) } return nil diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 6ad0cb773..8e3a7cce4 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -96,6 +96,11 @@ func TestDiagnosticsVersion_Compare(t *testing.T) { if err != nil { t.Fatalf("Versions should match") } + d.SetVersion("v1.7.0") + err = d.CompareVersion("0.7.2") + if err != nil { + t.Fatalf("Local version is greater") + } } func TestDiagnosticsVersion_Check(t *testing.T) { diff --git a/docs/administration.md b/docs/administration.md index 23b5d17f0..795d6f1c4 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -55,6 +55,18 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` +##### Importing Field Values + +If you are using [BSI Range-Encoding](../data-model/#bsi-range-encoding) field values, you can import field values for a single frame and single field using `--field`. The CSV file should be in the format `ColumnID,Value`. + +``` +pilosa import -i project -f stargazer --field star_count project-stargazer-counts.csv +``` + +
+

Note that you must first create a frame with Range Encoding enabled and a field. View Create Frame for more details.

+
+ #### Exporting Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format RowID,ColumnID and sorted by column ID. @@ -116,21 +128,21 @@ Note: This will only work when the replication factor is >= 2 Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions. -Version: Version string of the build. -Host: Host URI. -Cluster: List of nodes in the Cluster. -NumNodes: Number of nodes in the Cluster. -NumCPU: Number of Cores per Node -BSIEnabled: Bit Slice Index Frames in use. -TimeQuantumEnabled: Time Quantum Frames in use. -InverseEnabled: Inverse Frames in use. -NumIndexes: Number of Indexes in the Cluster. -NumFrames: Number of Frames in the Cluster. -NumSlices: Number of Slices in the Cluster. -NumViews: Number of Views in the Cluster. -OpenFiles: Open file handle count. -GoRoutines: Go routine count. - +- **Version:** Version string of the build. +- **Host:** Host URI. +- **Cluster:** List of nodes in the Cluster. +- **NumNodes:** Number of nodes in the Cluster. +- **NumCPU:** Number of Cores per Node +- **BSIEnabled:** Bit Slice Index Frames in use. +- **TimeQuantumEnabled:** Time Quantum Frames in use. +- **InverseEnabled:** Inverse Frames in use. +- **NumIndexes:** Number of Indexes in the Cluster. +- **NumFrames:** Number of Frames in the Cluster. +- **NumSlices:** Number of Slices in the Cluster. +- **NumViews:** Number of Views in the Cluster. +- **OpenFiles:** Open file handle count. +- **GoRoutines:** Go routine count. + You can opt-out of the Pilosa diagnostics reporting by setting either the command line configuration option `--metric.diagnostics=false`, use the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option. #### Metrics @@ -154,42 +166,23 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following: ##### Events We currently track the following events -Index: The creation of a new Index. - -Frame: The creation of a new Frame. - -MaxSlice: The Creation of a new Slice. - -SetBit: Count of set bits. - -ClearBit: Count of cleared bits. - -ImportBit: During a bulk data import this represents the count of bits created. - -SetRowAttrs: Count of Attributes set per row. - -SetColumnAttrs: Count of Attributes set per collumn. - -Bitmap: Count of Bitmap queries. - -TopN: Count of TopN queries. - -Union: Count of Union queries. - -Intersection: Count of Intersection queries. - -Difference: Count of Difference queries. - -Count: Count of Count queries. - -Range: Count of Range queries. - -Snapshot: Event count when the snapshot process is triggered. - -BlockRepair: Count of data blocks that were out of sync and repaired. - -Garbage Collection: Event count when Garbage Collection occurs. - -Goroutines: Number of running Goroutines. - -OpenFiles: Number of open file handles associated with running Pilosa process ID. +- **Index:** The creation of a new Index. +- **Frame:** The creation of a new Frame. +- **MaxSlice:** The Creation of a new Slice. +- **SetBit:** Count of set bits. +- **ClearBit:** Count of cleared bits. +- **ImportBit:** During a bulk data import this represents the count of bits created. +- **SetRowAttrs:** Count of Attributes set per row. +- **SetColumnAttrs:** Count of Attributes set per collumn. +- **Bitmap:** Count of Bitmap queries. +- **TopN:** Count of TopN queries. +- **Union:** Count of Union queries. +- **Intersection:** Count of Intersection queries. +- **Difference:** Count of Difference queries. +- **Count:** Count of Count queries. +- **Range:** Count of Range queries. +- **Snapshot:** Event count when the snapshot process is triggered. +- **BlockRepair:** Count of data blocks that were out of sync and repaired. +- **Garbage Collection:** Event count when Garbage Collection occurs. +- **Goroutines:** Number of running Goroutines. +- **OpenFiles:** Number of open file handles associated with running Pilosa process ID. diff --git a/docs/client-libraries.md b/docs/client-libraries.md index cad26a18f..67e36b5dc 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -46,18 +46,18 @@ func main() { } // We need to refer to indexes and frames before we can use them in a query. - repository, _ := schema.Index("repository", nil) - stargazer, _ := repository.Frame("stargazer", nil) - language, _ := repository.Frame("language", nil) + repository, _ := schema.Index("repository") + stargazer, _ := repository.Frame("stargazer") + language, _ := repository.Frame("language") var response *pilosa.QueryResponse // Which repositories did user 14 star: - response, _ = client.Query(stargazer.Bitmap(14), nil) + response, _ = client.Query(stargazer.Bitmap(14)) fmt.Println("User 14 starred: ", response.Result().Bitmap.Bits) // What are the top 5 languages in the sample data? - response, err = client.Query(language.TopN(5), nil) + response, err = client.Query(language.TopN(5)) languageIDs := []uint64{} for _, item := range response.Result().CountItems { languageIDs = append(languageIDs, item.ID) @@ -68,16 +68,14 @@ func main() { response, _ = client.Query( repository.Intersect( stargazer.Bitmap(14), - stargazer.Bitmap(19)), - nil) + stargazer.Bitmap(19))) fmt.Println("Both user 14 and 19 starred:", response.Result().Bitmap.Bits) // Which repositories were starred by user 14 or 19: response, _ = client.Query( repository.Union( stargazer.Bitmap(14), - stargazer.Bitmap(19)), - nil) + stargazer.Bitmap(19))) fmt.Println("User 14 or 19 starred:", response.Result().Bitmap.Bits) // Which repositories were starred by user 14 or 19 and were written in language 1: @@ -87,12 +85,11 @@ func main() { stargazer.Bitmap(14), stargazer.Bitmap(19), ), - language.Bitmap(1), - ), nil) + language.Bitmap(1))) fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Bitmap.Bits) // Set user 99999 as a stargazer for repository 77777? - client.Query(stargazer.SetBit(99999, 77777), nil) + client.Query(stargazer.SetBit(99999, 77777)) } ``` diff --git a/docs/configuration.md b/docs/configuration.md index 5cc7209b1..4fe15d855 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -75,6 +75,28 @@ Any flag that has a value that is a comma separated list on the command line bec data-dir = "~/.pilosa" ``` +#### Log Path + +* Description: Path of log file. +* Flag: `--log-path="/path/to/logfile"` +* Env: `PILOSA_LOG_PATH="/path/to/logfile"` +* Config: + + ```toml + log_path = "/path/to/logfile" + ``` + +#### Max Writes Per Request + +* Description: Maximum number of mutating commands allowed per request. This includes SetBit, ClearBit, SetRowAttrs, SetColumnAttrs, and SetFieldValue. +* Flag: `--max-writes-per-request=5000` +* Env: `PILOSA_MAX_WRITES_PER_REQUEST=5000` +* Config: + + ```toml + max-writes-per-request = 5000 + ``` + #### Gossip Port * Description: Port to which Pilosa should bind for internal communication. @@ -122,6 +144,30 @@ Any flag that has a value that is a comma separated list on the command line bec hosts = ["localhost:10101"] ``` +#### Cluster Poll Interval + +* Description: Polling interval for cluster. +* Flag: `cluster.poll-interval="1m0s"` +* Env: `PILOSA_CLUSTER_POLL_INTERVAL="1m0s"` +* Config: + + ```toml + [cluster] + poll-interval = "1m0s" + ``` + +#### Cluster Long Query Time + +* Description: Duration that will trigger log and stat messages for slow queries. +* Flag: `cluster.long-query-time="1m0s"` +* Env: `PILOSA_CLUSTER_LONG_QUERY_TIME="1m0s"` +* Config: + + ```toml + [cluster] + long-query-time = "1m0s" + ``` + #### Cluster Replicas * Description: Number of hosts each piece of data should be stored on. diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 000000000..bf5e76f29 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,347 @@ ++++ +title = "Examples" +weight = 4 +nav = [ + "Transportation", + "Chemical similarity search", +] ++++ + +## Examples + +### Transportation + +#### Introduction + +New York City released an extremely detailed data set of over 1 billion taxi rides taken in the city - this data has become a popular target for analysis by tech bloggers and has been very well studied. For this reason, we thought it would be interesting to import this data to Pilosa in order to compare with other data stores and techniques on the exact same data set. + +Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions). + +We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step. + +After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa. + +#### Data Model + +The NYC taxi data is comprised of a number of csv files listed here: http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml. These data files have around 20 columns, about half of which are relevant to the benchmark queries we're looking at: + +* Distance: miles, floating point +* Fare: dollars, floating point +* Number of passengers: integer +* Dropoff location: latitude and longitude, floating point +* Pickup location: latitude and longitude, floating point +* Dropoff time: timestamp +* Pickup time: timestamp + +We import these fields, creating one or more Pilosa frames from each of them: + +frame |mapping +------------|--------------------- +cab_type |direct map of enum int → row ID +dist_miles |round(dist) → row ID +total_amount_dollars |round(dist) → row ID +passenger_count |direct map of integer value → row ID +drop_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID +drop_year |year(timestamp) → row ID +drop_month |month(timestamp) → row ID +drop_day |day(timestamp) → row ID +drop_time |time of day mapped to one of 48 half-hour buckets +pickup_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID +pickup_year |year(timestamp) → row ID +pickup_month |month(timestamp) → row ID +pickup_day |day(timestamp) → row ID +pickup_time |time of day mapped to one of 48 half-hour buckets → row ID + +We also created two extra frames that represent the duration and average speed of each ride: + +frame |mapping +--------------------|------------- +duration_minutes |round(drop_timestamp - pickup_timestamp) → row ID +speed_mph |round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID + +#### Mapping + +Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities. + +##### 0 columns → 1 frame + +cab_type: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant. + +##### 1 column → 1 frame + +The following three frames are mapped in a simple direct way from single columns of the original data. + +dist_miles: each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times. + +In PDK parlance, we define a Mapper, which is simply a function that returns integer row IDs. PDK has a number of predefined mappers that can be described with a few parameters. One of these is LinearFloatMapper, which applies a linear function to the input, and casts it to an integer, so the rounding is handled implicitly. In code: +```go +lfm := pdk.LinearFloatMapper{ + Min: -0.5, + Max: 3600.5, + Res: 3601, +} +``` + +`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three. + +This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`). +```go +pdk.BitMapper{ + Frame: "dist_miles", + Mapper: lfm, + Parsers: []pdk.Parser{pdk.FloatParser{}}, + Fields: []int{fields["trip_distance"]}, +}, +``` + +These same objects are represented in the JSON definition file: +```go +{ + "Fields": { + "Trip_distance": 10 + }, + "Mappers": [ + { + "Name": "lfm0", + "Min": -0.5, + "Max": 3600.5, + "Res": 3600 + } + ], + "BitMappers": [ + { + "Frame": "dist_miles", + "Mapper": { + "Name": "lfm0" + }, + "Parsers": [ + {"Name": "FloatParser"} + ], + "Fields": "Trip_distance" + } + ] +} +``` + +Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of BitMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names to column indices. We use these names in the BitMapper definitions to keep things human-readable. + +**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The BitMapper definition is very similar to the previous one. + +**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID. + +##### 1 column → multiple frames + +When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis. + +We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day". + +We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day". + +We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time. + +##### Multiple columns → 1 frame + +The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID. + +We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id. + +Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient. + +##### Complex mappings + +We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work: +```go +durm := pdk.CustomMapper{ + Func: func(fields ...interface{}) interface{} { + start := fields[0].(time.Time) + end := fields[1].(time.Time) + return end.Sub(start).Minutes() + }, + Mapper: lfm, +} +``` + +#### Import process + +After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process. + +#### Queries + +Now we can run some example queries. + +Count per cab type can be retrieved, sorted, with a single PQL call. + +``` +TopN(frame=cab_type) +``` + +High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs. + +``` +TopN(frame=pickup_grid_id) +``` + +Average of total_amount per passenger_count can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average. + +```python +queries = '' +pcounts = range(10) +for i in pcounts: + queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i +resp = requests.post(qurl, data=queries) + +average_amounts = [] +for pcount, topn in zip(pcounts, resp.json()['results']): + wsum = sum([r['count'] * r['key'] for r in topn]) + count = sum([r['count'] for r in topn]) + average_amounts.append(float(wsum)/count) +``` + +For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb). + +### Chemical similarity search + +#### Overview + +The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals. + +We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set. + +Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient. +``` +T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B)) +``` + +A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical. + +All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase + +#### Data model + +We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. + +Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions. + +Given a SMILES encoded molecule and a similarity threshold, we want to retrieve all molecule ids (or SMILES) that have a similarity percentage greater than or equal to the similarity threshold. For example, given a molecule with: +``` +SMILES = "IC=C1/CCC(C(=O)O1)c2cccc3ccccc23" +threshold = 90 +``` + +return the set of molecules that have at least a 90% similarity with the given molecule. + +The Inverse view swaps the rows and columns automatically to enable queries over either the chembl_id or fingerprint. + +Standard View is used to calculate similarity +``` +Index: mole + View: Standard + Col: chembl_id + Frame: fingerprint + Row: position_id ("on" bit positions of a fingerprint) +``` + +Inverse View is used for finding chembl_id based on given SMILES. +From a given SMILES, we use RDKit to convert it to fingerprints with "on" bit position. From "on" bit positions, we can search a list of chembl_ids that match the bit positions. To choose the right chembl_id, we need another query to Standard View then choose the right chembl_id which has the length that matches the given fingerprint's length after using RDKit to convert SMILES to fingerprint. +``` +Index: mole + View: Inverse + Col: position_id ("on" bit positions of a fingerprint) + Frame: fingerprint + Row: chembl_id +``` + +After retrieving chembl_id from the Inverse View, we can use the Tanimoto coefficient to compare chembl_id with the entire data set of molecules. The result of this comparison is the list of `chembl_id`s that have a Tanimoto coefficient greater than the given threshold. + +#### Import process + +To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key. + +For the `mole` index, each row in the csv file has the format 'chembl_id, position_id' by running the following command from Chem-usecase: +``` +python import_from_sdf.py -p -file id_fingerprint.csv +``` + + +First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. +The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393 +``` +curl localhost:10101/index/mole \ + -X POST \ + -d '{"options": {"columnLabel": "position_id"}}' + +curl localhost:10101/index/mole/frame/fingerprint \ + -X POST \ + -d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}' + +``` + +Run the following commands to import the csv data into the `mole` index: +``` +pilosa import -d mole -f fingerprint id_fingerprint.csv +``` + +#### Queries + +Get chembl_id from a given SMILES: +``` +python get_mol_fr_smile.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" +``` + +Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get all chemlb_id that have positions are on, which following these steps: + +* Convert SMILES to fingerprint bit "on" positions + + ```python + from rdkit import Chem + from rdkit.Chem import AllChem + mol=Chem.MolFromSmiles("I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23") + fp = list(AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=4096).GetOnBits()) + ``` + +* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id + + ```python + bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] + bitmap_string = ', '.join(bit_maps) + intersection = "Intersect(%s)" % bitmap_string + mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"] + ``` + +* From list of chembl_id, query all "on" position from mol index, if the length of array of "on" position is matched to len(fp) then return that chembl_id, otherwise the given SMILES does not exist. + + ```python + for m in mole_ids: + mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"] + existed_mol = False + if len(mol) == len(fp): + found = m + existed_mol = True + break + ``` + +Retrieve molecule_ids that have similarity with SMILES="I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" and similarity threshold = 70% +``` +python similar.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" -t 70 +``` + +Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN query to get all chemlb_id that have position is on, which following these steps: + +* Get chembl_id from a SMILES (steps discussed above) + +* Query Pilosa’s TopN to get list of similarity chembl_id + ```python + query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)' + topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string) + ``` + +#### Benchmark + +To run benchmark for specific chembl_id for different similarity threshold at percentage of [50, 70, 75, 80, 85, 90], run following command: +``` +python benchmarks.py -id 6223 +``` + +As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa. + +Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster diff --git a/docs/getting-started.md b/docs/getting-started.md index 84c5fd0bf..3a0306a51 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -100,7 +100,7 @@ docker cp language.csv pilosa:/language.csv docker exec -it pilosa /pilosa import -i repository -f language /language.csv ``` -Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `language.txt` to see the mapping for languages. +Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `languages.txt` to see the mapping for languages. ### Input Definition Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data. @@ -155,4 +155,4 @@ curl localhost:10101/index/repository/query \ ### What's Next? -You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Tutorials](../tutorials/) for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/). +You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Examples](../examples/) page for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/). diff --git a/docs/howtos.md b/docs/howtos.md deleted file mode 100644 index 7809b7091..000000000 --- a/docs/howtos.md +++ /dev/null @@ -1,214 +0,0 @@ -+++ -title = "How Tos" -weight = 4 -nav = [ - "how-to-setup-a-secure-pilosa-cluster" -] -+++ - -## How Tos - -### How To Setup a Secure Pilosa Cluster - -#### Introduction - -Pilosa supports encrypting the communication between and to nodes in a cluster using TLS. In this tutorial, we will be setting up a three node Pilosa cluster running on the same computer. The same steps can be used for a multi-computer cluster but that requires setting up firewalls and other platform-specific configuration which is out of the scope of this tutorial. - -This tutorial assumes that you are using a UNIX-like system, such as Linux or MacOS. [Windows Subsystem for Linux (WSL)](https://msdn.microsoft.com/en-us/commandline/wsl/about) works equally well on Windows 10 systems. - -#### Installing Pilosa and Creating the Directory Structure - -If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](https://www.pilosa.com/docs/latest/installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](https://www.pilosa.com/docs/latest/installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](https://www.pilosa.com/docs/latest/installation/#build-from-source). - -After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line: -``` -pilosa --help -``` - -Let's create a directory for the tutorial to put all of our files and switch to that directory: -``` -mkdir $HOME/pilosa-tls-tutorial && cd $_ -``` - -#### Creating the TLS Certificate and Gossip Key - -Securing a Pilosa cluster consists of securing the communication between nodes using TLS and Gossip encryption. [Pilosa Enterprise](https://www.pilosa.com/enterprise/) additionally supports authentication and other security features, but those are not covered in this tutorial. - -The first step is acquiring an SSL certificate. You can buy a commercial certificate or retrieve a Let's Encrypt certificiate but we will be using a self signed certificate for practical reasons. Using self-signed certificates is not recommended in production, since it makes man in the middle attacks easy. - -The following command creates a 2048bit self-signed wildcard certificate for `*.pilosa.local` which expires 10 years later. - -``` -openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.crt -days 3650 -nodes -subj "/C=US/ST=Texas/L=Austin/O=Pilosa/OU=Com/CN=*.pilosa.local" -``` - -The command above creates two files in the current directory: -* `pilosa.local.crt` is the SSL certificate. -* `pilosa.local.key` is the private key file which must be kept as secret. - -Having created the SSL certificate, we can now create the gossip encryption key. Gossip encryption key file must be exactly 16, 24, or 32 bytes to select one of AES-128, AES-192, or AES-256 encryption. Reading random bytes from cryptographically secure `/dev/random` serves our purpose very well: -``` -head -c 32 /dev/random > pilosa.local.gossip32 -``` - -We now should have `pilosa.local.gossip32` in the current directory with 32 random bytes. - -#### Creating the Configuration Files - -Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes. - -Create `node1.config.toml` in the project directory and paste the following in it: - -```toml -# node1.config.toml - -data-dir = "node1_data" -bind = "https://01.pilosa.local:10501" - -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - -[tls] -certificate = "pilosa.local.crt" -key = "pilosa.local.key" -skip-verify = true - -[gossip] -seed = "01.pilosa.local:15000" -port = 15000 -key = "pilosa.local.gossip32" -``` - -Create `node2.config.toml` in the project directory and paste the following in it: - -```toml -# node2.config.toml - -data-dir = "node2_data" -bind = "https://02.pilosa.local:10502" - -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - -[tls] -certificate = "pilosa.local.crt" -key = "pilosa.local.key" -skip-verify = true - -[gossip] -seed = "01.pilosa.local:15000" -port = 16000 -key = "pilosa.local.gossip32" -``` - -Create `node3.config.toml` in the project directory and paste the following in it: - -```toml -# node3.config.toml - -data-dir = "node3_data" -bind = "https://03.pilosa.local:10503" - -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - -[tls] -certificate = "pilosa.local.crt" -key = "pilosa.local.key" -skip-verify = true - -[gossip] -seed = "01.pilosa.local:15000" -port = 17000 -key = "pilosa.local.gossip32" -``` - -Here is some explanation of the configuration items: -* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it. -* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes. -* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings. -* `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers. -* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. - -#### Final Touches Before Running the Cluster - -Before running the cluster, let's make sure that `01.pilosa.local`, `02.pilosa.local` and `03.pilosa.local` resolve to an IP address. If you are running the cluster on your computer, it is adequate to add them to your `/etc/hosts`. Below is one of the many ways of doing that (mind the `>>`): -``` -sudo sh -c 'printf "\n127.0.0.1 01.pilosa.local 02.pilosa.local 03.pilosa.local\n" >> /etc/hosts' -``` - -Ensure we can access the hosts in the cluster: -``` -ping -c 1 01.pilosa.local -ping -c 1 02.pilosa.local -ping -c 1 03.pilosa.local -``` - -If any of the commands above return `ping: unknown host`, make sure your `/etc/hosts` contains the failed hostname. - -#### Running the Cluster - -Let's open three terminal windows and run each node in its window. This will enable us to better observe what's happening on which node. - -Switch to the first terminal window, change to the project directory and start the first node: -``` -cd $HOME/pilosa-tls-tutorial -pilosa server -c node1.config.toml -``` - -Switch to the second terminal window, change to the project directory and start the second node: -``` -cd $HOME/pilosa-tls-tutorial -pilosa server -c node2.config.toml -``` - -Switch to the third terminal window, change to the project directory and start the third node: -``` -cd $HOME/pilosa-tls-tutorial -pilosa server -c node3.config.toml -``` - -Let's ensure that all three Pilosa servers are runnning and they are connected: -``` -curl -k --ipv4 https://01.pilosa.local:10501/status -``` - -The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL. - -All nodes should be in the `UP` state: -``` -{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}} -``` - -#### Running Queries - -Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame: -``` -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d '' -``` - -This will create index `sample-index` with default options. Let's create the frame now: -``` -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d '' -``` - -We just created frame `sample-frame` with default options. - -Let's run a `SetBit` query: -``` -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)' -``` - -Confirm that the bit was indeed set: -``` -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' -``` - -The same response should be returned when querying other nodes in the cluster: -``` -curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' -``` - -#### What's Next? - -Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa. diff --git a/docs/input-definition.md b/docs/input-definition.md index da59ee391..494f4757e 100644 --- a/docs/input-definition.md +++ b/docs/input-definition.md @@ -46,7 +46,7 @@ curl localhost:10101/index/repository/input-definition/stargazer \ { "name": "repo_id", "primaryKey": true - }, + }, { "actions": [ { @@ -105,8 +105,8 @@ curl localhost:10101/index/repository/input/stargazer \ -X POST \ -d '[ { - "language_id": "Go", "repo_id": 91720568, + "language_id": "Go", "stargazer_id": 513114, "time_value": "2017-05-18T20:40" }, diff --git a/docs/installation.md b/docs/installation.md index c0455ac32..33528f969 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -74,19 +74,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.7.1/pilosa-v0.7.1-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.0/pilosa-v0.8.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.7.1-darwin-amd64.tar.gz + tar xfz pilosa-v0.8.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.7.1-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.8.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -228,19 +228,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.7.1/pilosa-v0.7.1-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.0/pilosa-v0.8.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.7.1-linux-amd64.tar.gz + tar xfz pilosa-v0.8.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.7.1-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.8.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: diff --git a/docs/query-language.md b/docs/query-language.md index 2a8130736..f28257fe6 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -427,7 +427,7 @@ Returns `{{"attrs":{},"bits":[10]}` **Spec:** ``` -Range(, ) +Range(, ) ``` **Description:** @@ -441,7 +441,7 @@ Returns bits that are true for the comparison operator. **Examples:** In our source data, commitactivity was counted over the last year. -The following Range query returns all repositories having more than 100 commits. +The following greater-than Range query returns all repositories having more than 100 commits. ``` Range(frame="stats", commitactivity > 100) @@ -451,6 +451,25 @@ Returns `{{"attrs":{},"bits":[10]}` * bits are repositories which had at least 100 commits in the last year. +BSI range queries support the following operators: + + Operator | Name | Value +----------|-------------------------------|-------------------- + `>` | greater-than, GT | integer + `<` | less-than, LT | integer + `<=` | less-than-or-equal-to, LTE | integer + `>=` | greater-than-or-equal-to, GTE | integer + `==` | equal-to, EQ | integer + `!=` | not-equal-to, NEQ | integer or `null` + `><` | between, BETWEEN | [integer, integer] + +The `BETWEEN` query specifies an interval with both bounds, using `><` operator, and a two-element list containing the lower and upper bounds of the interval: + +``` +Range(frame="stats", commitactivity >< [100, 200]) +``` + +This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way. #### Sum @@ -499,4 +518,4 @@ SetFieldValue returns `null` upon success. Set the number of pull requests of repository 10. ``` SetFieldValue(col=10, frame="stats", pullrequests=2) -``` \ No newline at end of file +``` diff --git a/docs/tutorials.md b/docs/tutorials.md index d304dc80d..4c2cdd7bd 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -2,346 +2,213 @@ title = "Tutorials" weight = 4 nav = [ - "Transportation", - "Chemical similarity search", + "How To Setup a Secure Cluster", ] +++ ## Tutorials -### Transportation +### How To Setup a Secure Cluster #### Introduction -New York City released an extremely detailed data set of over 1 billion taxi rides taken in the city - this data has become a popular target for analysis by tech bloggers and has been very well studied. For this reason, we thought it would be interesting to import this data to Pilosa in order to compare with other data stores and techniques on the exact same data set. +Pilosa supports encrypting the communication between and to nodes in a cluster using TLS. In this tutorial, we will be setting up a three node Pilosa cluster running on the same computer. The same steps can be used for a multi-computer cluster but that requires setting up firewalls and other platform-specific configuration which is out of the scope of this tutorial. -Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions). +This tutorial assumes that you are using a UNIX-like system, such as Linux or MacOS. [Windows Subsystem for Linux (WSL)](https://msdn.microsoft.com/en-us/commandline/wsl/about) works equally well on Windows 10 systems. -We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step. +#### Installing Pilosa and Creating the Directory Structure -After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa. +If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](https://www.pilosa.com/docs/latest/installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](https://www.pilosa.com/docs/latest/installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](https://www.pilosa.com/docs/latest/installation/#build-from-source). -#### Data Model - -The NYC taxi data is comprised of a number of csv files listed here: http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml. These data files have around 20 columns, about half of which are relevant to the benchmark queries we're looking at: - -* Distance: miles, floating point -* Fare: dollars, floating point -* Number of passengers: integer -* Dropoff location: latitude and longitude, floating point -* Pickup location: latitude and longitude, floating point -* Dropoff time: timestamp -* Pickup time: timestamp - -We import these fields, creating one or more Pilosa frames from each of them: - -frame |mapping -------------|--------------------- -cab_type |direct map of enum int → row ID -dist_miles |round(dist) → row ID -total_amount_dollars |round(dist) → row ID -passenger_count |direct map of integer value → row ID -drop_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID -drop_year |year(timestamp) → row ID -drop_month |month(timestamp) → row ID -drop_day |day(timestamp) → row ID -drop_time |time of day mapped to one of 48 half-hour buckets -pickup_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID -pickup_year |year(timestamp) → row ID -pickup_month |month(timestamp) → row ID -pickup_day |day(timestamp) → row ID -pickup_time |time of day mapped to one of 48 half-hour buckets → row ID - -We also created two extra frames that represent the duration and average speed of each ride: - -frame |mapping ---------------------|------------- -duration_minutes |round(drop_timestamp - pickup_timestamp) → row ID -speed_mph |round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID - -#### Mapping - -Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities. - -##### 0 columns → 1 frame - -cab_type: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant. - -##### 1 column → 1 frame - -The following three frames are mapped in a simple direct way from single columns of the original data. - -dist_miles: each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times. - -In PDK parlance, we define a Mapper, which is simply a function that returns integer row IDs. PDK has a number of predefined mappers that can be described with a few parameters. One of these is LinearFloatMapper, which applies a linear function to the input, and casts it to an integer, so the rounding is handled implicitly. In code: -```go -lfm := pdk.LinearFloatMapper{ - Min: -0.5, - Max: 3600.5, - Res: 3601, -} +After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line: +``` +pilosa --help ``` -`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three. - -This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`). -```go -pdk.BitMapper{ - Frame: "dist_miles", - Mapper: lfm, - Parsers: []pdk.Parser{pdk.FloatParser{}}, - Fields: []int{fields["trip_distance"]}, -}, +Let's create a directory for the tutorial to put all of our files and switch to that directory: +``` +mkdir $HOME/pilosa-tls-tutorial && cd $_ ``` -These same objects are represented in the JSON definition file: -```go -{ - "Fields": { - "Trip_distance": 10 - }, - "Mappers": [ - { - "Name": "lfm0", - "Min": -0.5, - "Max": 3600.5, - "Res": 3600 - } - ], - "BitMappers": [ - { - "Frame": "dist_miles", - "Mapper": { - "Name": "lfm0" - }, - "Parsers": [ - {"Name": "FloatParser"} - ], - "Fields": "Trip_distance" - } - ] -} -``` +#### Creating the TLS Certificate and Gossip Key -Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of BitMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names to column indices. We use these names in the BitMapper definitions to keep things human-readable. +Securing a Pilosa cluster consists of securing the communication between nodes using TLS and Gossip encryption. [Pilosa Enterprise](https://www.pilosa.com/enterprise/) additionally supports authentication and other security features, but those are not covered in this tutorial. -**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The BitMapper definition is very similar to the previous one. +The first step is acquiring an SSL certificate. You can buy a commercial certificate or retrieve a Let's Encrypt certificiate but we will be using a self signed certificate for practical reasons. Using self-signed certificates is not recommended in production, since it makes man in the middle attacks easy. -**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID. - -##### 1 column → multiple frames - -When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis. - -We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day". - -We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day". - -We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time. - -##### Multiple columns → 1 frame - -The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID. - -We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id. - -Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient. - -##### Complex mappings - -We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work: -```go -durm := pdk.CustomMapper{ - Func: func(fields ...interface{}) interface{} { - start := fields[0].(time.Time) - end := fields[1].(time.Time) - return end.Sub(start).Minutes() - }, - Mapper: lfm, -} -``` - -#### Import process - -After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process. - -#### Queries - -Now we can run some example queries. - -Count per cab type can be retrieved, sorted, with a single PQL call. +The following command creates a 2048bit self-signed wildcard certificate for `*.pilosa.local` which expires 10 years later. ``` -TopN(frame=cab_type) +openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.crt -days 3650 -nodes -subj "/C=US/ST=Texas/L=Austin/O=Pilosa/OU=Com/CN=*.pilosa.local" ``` -High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs. +The command above creates two files in the current directory: +* `pilosa.local.crt` is the SSL certificate. +* `pilosa.local.key` is the private key file which must be kept as secret. +Having created the SSL certificate, we can now create the gossip encryption key. Gossip encryption key file must be exactly 16, 24, or 32 bytes to select one of AES-128, AES-192, or AES-256 encryption. Reading random bytes from cryptographically secure `/dev/random` serves our purpose very well: ``` -TopN(frame=pickup_grid_id) +head -c 32 /dev/random > pilosa.local.gossip32 ``` -Average of total_amount per passenger_count can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average. +We now should have `pilosa.local.gossip32` in the current directory with 32 random bytes. -```python -queries = '' -pcounts = range(10) -for i in pcounts: - queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i -resp = requests.post(qurl, data=queries) +#### Creating the Configuration Files -average_amounts = [] -for pcount, topn in zip(pcounts, resp.json()['results']): - wsum = sum([r['count'] * r['key'] for r in topn]) - count = sum([r['count'] for r in topn]) - average_amounts.append(float(wsum)/count) +Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes. + +Create `node1.config.toml` in the project directory and paste the following in it: + +```toml +# node1.config.toml + +data-dir = "node1_data" +bind = "https://01.pilosa.local:10501" + +[cluster] +hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] + +[tls] +certificate = "pilosa.local.crt" +key = "pilosa.local.key" +skip-verify = true + +[gossip] +seed = "01.pilosa.local:15000" +port = 15000 +key = "pilosa.local.gossip32" ``` -For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb). +Create `node2.config.toml` in the project directory and paste the following in it: -### Chemical similarity search +```toml +# node2.config.toml -#### Overview +data-dir = "node2_data" +bind = "https://02.pilosa.local:10502" -The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals. +[cluster] +hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] -We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set. +[tls] +certificate = "pilosa.local.crt" +key = "pilosa.local.key" +skip-verify = true -Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient. -``` -T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B)) +[gossip] +seed = "01.pilosa.local:15000" +port = 16000 +key = "pilosa.local.gossip32" ``` -A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical. +Create `node3.config.toml` in the project directory and paste the following in it: -All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase +```toml +# node3.config.toml -#### Data model +data-dir = "node3_data" +bind = "https://03.pilosa.local:10503" -We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. +[cluster] +hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] -Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions. +[tls] +certificate = "pilosa.local.crt" +key = "pilosa.local.key" +skip-verify = true -Given a SMILES encoded molecule and a similarity threshold, we want to retrieve all molecule ids (or SMILES) that have a similarity percentage greater than or equal to the similarity threshold. For example, given a molecule with: -``` -SMILES = "IC=C1/CCC(C(=O)O1)c2cccc3ccccc23" -threshold = 90 +[gossip] +seed = "01.pilosa.local:15000" +port = 17000 +key = "pilosa.local.gossip32" ``` -return the set of molecules that have at least a 90% similarity with the given molecule. +Here is some explanation of the configuration items: +* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it. +* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes. +* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings. +* `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers. +* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. -The Inverse view swaps the rows and columns automatically to enable queries over either the chembl_id or fingerprint. +#### Final Touches Before Running the Cluster -Standard View is used to calculate similarity +Before running the cluster, let's make sure that `01.pilosa.local`, `02.pilosa.local` and `03.pilosa.local` resolve to an IP address. If you are running the cluster on your computer, it is adequate to add them to your `/etc/hosts`. Below is one of the many ways of doing that (mind the `>>`): ``` -Index: mole - View: Standard - Col: chembl_id - Frame: fingerprint - Row: position_id ("on" bit positions of a fingerprint) +sudo sh -c 'printf "\n127.0.0.1 01.pilosa.local 02.pilosa.local 03.pilosa.local\n" >> /etc/hosts' ``` -Inverse View is used for finding chembl_id based on given SMILES. -From a given SMILES, we use RDKit to convert it to fingerprints with "on" bit position. From "on" bit positions, we can search a list of chembl_ids that match the bit positions. To choose the right chembl_id, we need another query to Standard View then choose the right chembl_id which has the length that matches the given fingerprint's length after using RDKit to convert SMILES to fingerprint. +Ensure we can access the hosts in the cluster: ``` -Index: mole - View: Inverse - Col: position_id ("on" bit positions of a fingerprint) - Frame: fingerprint - Row: chembl_id +ping -c 1 01.pilosa.local +ping -c 1 02.pilosa.local +ping -c 1 03.pilosa.local ``` -After retrieving chembl_id from the Inverse View, we can use the Tanimoto coefficient to compare chembl_id with the entire data set of molecules. The result of this comparison is the list of `chembl_id`s that have a Tanimoto coefficient greater than the given threshold. +If any of the commands above return `ping: unknown host`, make sure your `/etc/hosts` contains the failed hostname. -#### Import process +#### Running the Cluster -To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key. +Let's open three terminal windows and run each node in its window. This will enable us to better observe what's happening on which node. -For the `mole` index, each row in the csv file has the format 'chembl_id, position_id' by running the following command from Chem-usecase: +Switch to the first terminal window, change to the project directory and start the first node: ``` -python import_from_sdf.py -p -file id_fingerprint.csv +cd $HOME/pilosa-tls-tutorial +pilosa server -c node1.config.toml ``` - -First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. -The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393 +Switch to the second terminal window, change to the project directory and start the second node: ``` -curl localhost:10101/index/mole \ - -X POST \ - -d '{"options": {"columnLabel": "position_id"}}' - -curl localhost:10101/index/mole/frame/fingerprint \ - -X POST \ - -d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}' - +cd $HOME/pilosa-tls-tutorial +pilosa server -c node2.config.toml ``` -Run the following commands to import the csv data into the `mole` index: +Switch to the third terminal window, change to the project directory and start the third node: ``` -pilosa import -d mole -f fingerprint id_fingerprint.csv +cd $HOME/pilosa-tls-tutorial +pilosa server -c node3.config.toml ``` -#### Queries - -Get chembl_id from a given SMILES: +Let's ensure that all three Pilosa servers are runnning and they are connected: ``` -python get_mol_fr_smile.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" +curl -k --ipv4 https://01.pilosa.local:10501/status ``` -Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get all chemlb_id that have positions are on, which following these steps: +The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL. -* Convert SMILES to fingerprint bit "on" positions - - ```python - from rdkit import Chem - from rdkit.Chem import AllChem - mol=Chem.MolFromSmiles("I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23") - fp = list(AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=4096).GetOnBits()) - ``` - -* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id - - ```python - bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] - bitmap_string = ', '.join(bit_maps) - intersection = "Intersect(%s)" % bitmap_string - mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"] - ``` - -* From list of chembl_id, query all "on" position from mol index, if the length of array of "on" position is matched to len(fp) then return that chembl_id, otherwise the given SMILES does not exist. - - ```python - for m in mole_ids: - mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"] - existed_mol = False - if len(mol) == len(fp): - found = m - existed_mol = True - break - ``` - -Retrieve molecule_ids that have similarity with SMILES="I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" and similarity threshold = 70% +All nodes should be in the `UP` state: ``` -python similar.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" -t 70 +{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}} ``` -Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN query to get all chemlb_id that have position is on, which following these steps: +#### Running Queries -* Get chembl_id from a SMILES (steps discussed above) - -* Query Pilosa’s TopN to get list of similarity chembl_id - ```python - query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)' - topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string) - ``` - -#### Benchmark - -To run benchmark for specific chembl_id for different similarity threshold at percentage of [50, 70, 75, 80, 85, 90], run following command: +Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame: ``` -python benchmarks.py -id 6223 +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d '' ``` -As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa. +This will create index `sample-index` with default options. Let's create the frame now: +``` +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d '' +``` -Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster +We just created frame `sample-frame` with default options. + +Let's run a `SetBit` query: +``` +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)' +``` + +Confirm that the bit was indeed set: +``` +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' +``` + +The same response should be returned when querying other nodes in the cluster: +``` +curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' +``` + +#### What's Next? + +Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa. diff --git a/executor.go b/executor.go index 7610747fa..322fc6aaf 100644 --- a/executor.go +++ b/executor.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "net/http" "sort" "time" @@ -50,12 +51,9 @@ type Executor struct { } // NewExecutor returns a new instance of Executor. -func NewExecutor(clientOptions *ClientOptions) *Executor { - if clientOptions == nil { - clientOptions = &ClientOptions{} - } +func NewExecutor(remoteClient *http.Client) *Executor { return &Executor{ - client: NewInternalHTTPClientFromURI(nil, clientOptions), + client: NewInternalHTTPClientFromURI(nil, remoteClient), } } @@ -967,7 +965,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql } // Forward call to remote node otherwise. - if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { + if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { return false, err } else { ret = res[0].(bool) @@ -1073,7 +1071,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C } // Forward call to remote node otherwise. - if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { + if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { return false, err } else { ret = res[0].(bool) @@ -1140,7 +1138,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) + _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) resp <- err }(node) } @@ -1198,7 +1196,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) + _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) resp <- err }(node) } @@ -1285,7 +1283,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, index, &pql.Query{Calls: calls}, nil, opt) + _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil, opt) resp <- err }(node) } @@ -1344,7 +1342,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) + _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) resp <- err }(node) } @@ -1360,7 +1358,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p } // exec executes a PQL query remotely for a set of slices on a node. -func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { +func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { // Encode request object. pbreq := &internal.QueryRequest{ Query: q.String(), @@ -1506,7 +1504,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod if n.URI == e.URI { resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) } else if !opt.Remote { - results, err := e.exec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) + results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) if len(results) > 0 { resp.result = results[0] } diff --git a/fragment.go b/fragment.go index 2dd78783b..e2d964ded 100644 --- a/fragment.go +++ b/fragment.go @@ -28,6 +28,7 @@ import ( "io" "io/ioutil" "log" + "net/http" "os" "sort" "sync" @@ -1682,9 +1683,9 @@ func (h *blockHasher) WriteValue(v uint64) { type FragmentSyncer struct { Fragment *Fragment - URI URI - Cluster *Cluster - ClientOptions *ClientOptions + URI URI + Cluster *Cluster + RemoteClient *http.Client Closing <-chan struct{} } @@ -1719,8 +1720,7 @@ func (s *FragmentSyncer) SyncFragment() error { } // Retrieve remote blocks. - client := NewInternalHTTPClientFromURI(&node.URI, s.ClientOptions) - + client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice()) if err != nil && err != ErrFragmentNotFound { return err @@ -1796,8 +1796,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { return nil } - client := NewInternalHTTPClientFromURI(&node.URI, s.ClientOptions) - + client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) clients = append(clients, client) // Only sync the standard block. diff --git a/handler.go b/handler.go index 11b1f0cb9..f9b21b4e1 100644 --- a/handler.go +++ b/handler.go @@ -56,9 +56,9 @@ type Handler struct { StatusHandler StatusHandler // Local hostname & cluster configuration. - URI URI - Cluster *Cluster - ClientOptions *ClientOptions + URI URI + Cluster *Cluster + RemoteClient *http.Client Router *mux.Router NormalRouter *mux.Router @@ -1539,7 +1539,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) } // Create a client for the remote cluster. - client := NewInternalHTTPClientFromURI(host, h.ClientOptions) + client := NewInternalHTTPClientFromURI(host, h.RemoteClient) // Determine the maximum number of slices. maxSlices, err := client.MaxSliceByIndex(r.Context()) @@ -1793,8 +1793,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - // Validation the input definition with the current index's ColumnLabel. - if err := req.Validate(index.ColumnLabel()); err != nil { + if err := req.Validate(); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -2079,10 +2078,9 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, for _, field := range inputDef.Fields() { validFields[field.Name] = true if field.PrimaryKey { - columnLabel := field.Name - value, ok := req[columnLabel] + value, ok := req[field.Name] if !ok { - return nil, fmt.Errorf("columnLabel required") + return nil, fmt.Errorf("primary key does not exist") } rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float if !ok { diff --git a/handler_test.go b/handler_test.go index 05116ef8c..c1e0a4c0b 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1329,7 +1329,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { t.Fatalf("unexpected body: %s", body) } - // Eusure throwing error if primary field's name doesn't match columnLabel + // Ensure throwing error if there's no primary key hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"}) unmatchColumnBody := []byte(` { @@ -1343,8 +1343,17 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { }], "fields": [ { - "name": "columnID", - "primaryKey": true + "name": "foo", + "actions": [ + { + "frame": "cab-type", + "valueDestination": "mapping", + "valueMap": { + "Green": 1, + "Yellow": 2 + } + } + ] } ] }`) @@ -1353,7 +1362,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i1/input-definition/input1", bytes.NewBuffer(unmatchColumnBody))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionColumnLabel.Error()+"\n" { + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionHasPrimaryKey.Error()+"\n" { t.Fatalf("unexpected body: %s", body) } @@ -1697,7 +1706,7 @@ func TestInput_JSON(t *testing.T) { "distanceMiles": 8, "withPet": true }]`, - err: "columnLabel required"}, + err: "primary key does not exist"}, {json: `[{ "id": 1, "cabType": "yellow", diff --git a/holder.go b/holder.go index d5a08ec58..122dafd7a 100644 --- a/holder.go +++ b/holder.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "log" + "net/http" "os" "path/filepath" "sort" @@ -520,9 +521,9 @@ func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.Lstd type HolderSyncer struct { Holder *Holder - URI URI - Cluster *Cluster - ClientOptions *ClientOptions + URI URI + Cluster *Cluster + RemoteClient *http.Client // Signals that the sync should stop. Closing <-chan struct{} @@ -608,7 +609,7 @@ func (s *HolderSyncer) syncIndex(index string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) { - client := NewInternalHTTPClientFromURI(&node.URI, s.ClientOptions) + client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. @@ -650,7 +651,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) { - client := NewInternalHTTPClientFromURI(&node.URI, s.ClientOptions) + client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. @@ -700,11 +701,11 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err // Sync fragments together. fs := FragmentSyncer{ - Fragment: frag, - URI: s.URI, - Cluster: s.Cluster, - Closing: s.Closing, - ClientOptions: s.ClientOptions, + Fragment: frag, + URI: s.URI, + Cluster: s.Cluster, + Closing: s.Closing, + RemoteClient: s.RemoteClient, } if err := fs.SyncFragment(); err != nil { return err diff --git a/holder_test.go b/holder_test.go index 5121db568..e1bfe9d6a 100644 --- a/holder_test.go +++ b/holder_test.go @@ -45,6 +45,9 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrIndexPermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() @@ -95,6 +98,9 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrFramePermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() @@ -151,6 +157,9 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrViewPermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() @@ -172,6 +181,9 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrViewFragmentsMkdir", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() @@ -194,6 +206,9 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrFragmentStoragePermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() @@ -240,6 +255,9 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrFragmentCachePermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() @@ -387,7 +405,7 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { cluster := test.NewCluster(2) - + client := pilosa.GetHTTPClient(nil) // Create a local holder. hldr0 := test.MustOpenHolder() defer hldr0.Close() @@ -399,7 +417,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { defer s.Close() s.Handler.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e := pilosa.NewExecutor(client) e.Holder = hldr1.Holder e.URI = cluster.Nodes[1].URI e.Cluster = cluster @@ -468,9 +486,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Set up syncer. syncer := pilosa.HolderSyncer{ - Holder: hldr0.Holder, - URI: cluster.Nodes[0].URI, - Cluster: cluster, + Holder: hldr0.Holder, + URI: cluster.Nodes[0].URI, + Cluster: cluster, + RemoteClient: pilosa.GetHTTPClient(nil), } if err := syncer.SyncHolder(); err != nil { diff --git a/index.go b/index.go index ff00e66d4..fb445bb57 100644 --- a/index.go +++ b/index.go @@ -691,7 +691,8 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin for _, fr := range pb.Frames { opt := FrameOptions{ - RowLabel: fr.Meta.RowLabel, + // Deprecating row labels per #810. So, setting the default row label here. + RowLabel: DefaultRowLabel, InverseEnabled: fr.Meta.InverseEnabled, CacheType: fr.Meta.CacheType, CacheSize: fr.Meta.CacheSize, diff --git a/index_test.go b/index_test.go index 5e5dce218..23a820f2d 100644 --- a/index_test.go +++ b/index_test.go @@ -305,14 +305,14 @@ func TestIndex_CreateInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } else if inputDef.Frames()[0].Name != frames.Name { t.Fatalf("unexpected input definition frames %v", inputDef.Frames()) - } else if inputDef.Fields()[0].Name != fields.Name { + } else if inputDef.Fields()[0].Name != field.Name { t.Fatalf("unexpected input definition actions %v", inputDef.Fields()) } } diff --git a/input_definition.go b/input_definition.go index 98ca8ca09..f4571d0bb 100644 --- a/input_definition.go +++ b/input_definition.go @@ -92,6 +92,8 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { i.frames = append(i.frames, inputFrame) } + primaryKeyGiven := false + for _, field := range pb.Fields { var actions []Action for _, action := range field.InputDefinitionActions { @@ -103,6 +105,10 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { }) } + if field.PrimaryKey { + primaryKeyGiven = true + } + inputField := InputDefinitionField{ Name: field.Name, PrimaryKey: field.PrimaryKey, @@ -111,6 +117,10 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { i.fields = append(i.fields, inputField) } + if len(pb.Fields) > 0 && !primaryKeyGiven { + return ErrInputDefinitionHasPrimaryKey + } + return nil } @@ -257,7 +267,7 @@ type InputDefinitionInfo struct { } // Validate the InputDefinitionInfo data. -func (i *InputDefinitionInfo) Validate(columnLabel string) error { +func (i *InputDefinitionInfo) Validate() error { numPrimaryKey := 0 accountRowID := make(map[string]uint64) @@ -273,6 +283,9 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error { // Validate columnLabel and duplicate primaryKey. for _, field := range i.Fields { + if field.Name == "" { + return ErrInputDefinitionNameRequired + } for _, action := range field.Actions { if err := action.Validate(); err != nil { return err @@ -290,9 +303,6 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error { } if field.PrimaryKey { numPrimaryKey++ - if field.Name != columnLabel { - return ErrInputDefinitionColumnLabel - } } else if len(field.Actions) == 0 { return ErrInputDefinitionActionRequired } diff --git a/input_definition_test.go b/input_definition_test.go index f2daade2c..93d0e393d 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -110,14 +110,14 @@ func TestActionValidation(t *testing.T) { action := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} field := pilosa.InputDefinitionField{Name: "id", PrimaryKey: false, Actions: []pilosa.Action{action}} info := pilosa.InputDefinitionInfo{Fields: []pilosa.InputDefinitionField{field}} - err := info.Validate("id") + err := info.Validate() if err != pilosa.ErrInputDefinitionAttrsRequired { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionAttrsRequired, err) } frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } @@ -126,7 +126,7 @@ func TestActionValidation(t *testing.T) { action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if err != pilosa.ErrName { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrName, err) } @@ -135,23 +135,15 @@ func TestActionValidation(t *testing.T) { action = pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if err != pilosa.ErrFrameRequired { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrFrameRequired, err) } - action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} - field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("test") - if err != pilosa.ErrInputDefinitionColumnLabel { - t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionColumnLabel, err) - } - action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "x", PrimaryKey: false, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if err != pilosa.ErrInputDefinitionHasPrimaryKey { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionHasPrimaryKey, err) } @@ -159,7 +151,7 @@ func TestActionValidation(t *testing.T) { action = pilosa.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if !strings.Contains(err.Error(), "invalid ValueDestination") { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } @@ -167,7 +159,7 @@ func TestActionValidation(t *testing.T) { action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if err != pilosa.ErrInputDefinitionValueMap { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionValueMap, err) } @@ -177,7 +169,7 @@ func TestActionValidation(t *testing.T) { action1 := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field1 := pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action1}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field, field1}} - err = info.Validate("id") + err = info.Validate() if !strings.Contains(err.Error(), "duplicate rowID with other field") { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } @@ -185,7 +177,7 @@ func TestActionValidation(t *testing.T) { field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true} field1 = pilosa.InputDefinitionField{Name: "test", PrimaryKey: false} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field, field1}} - err = info.Validate("id") + err = info.Validate() if err != pilosa.ErrInputDefinitionActionRequired { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionActionRequired, err) } diff --git a/roaring/roaring.go b/roaring/roaring.go index 62f87d94b..97994c37a 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2508,17 +2508,25 @@ func differenceRunArray(a, b *container) *container { // differenceRunBitmap computes the difference of an run from a bitmap. func differenceRunBitmap(a, b *container) *container { // If a is full, difference is the flip of b. - if a.runs[0].start == 0 && a.runs[0].last == 65535 { + if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 { return b.flipBitmap() } output := &container{container_type: ContainerRun} output.n = a.n + if len(a.runs) == 0 { + return output + } for j := 0; j < len(a.runs); j++ { run := a.runs[j] + add := true for bit := a.runs[j].start; bit <= a.runs[j].last; bit++ { if b.bitmapContains(bit) { output.n-- if run.start == bit { + if bit == 65535 { //overflow + add = false + } + run.start++ } else if bit == run.last { run.last-- @@ -2534,10 +2542,15 @@ func differenceRunBitmap(a, b *container) *container { break } } + + if bit == 65535 { //overflow + break + } } if run.start <= run.last { - output.runs = append(output.runs, run) - + if add { + output.runs = append(output.runs, run) + } } } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index bdc08b649..b36723b26 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1485,6 +1485,12 @@ func MakeBitmap(start []uint64) []uint64 { } return b } +func MakeLastBitSet() []uint64 { + obj := NewBitmap(65535) + c := obj.container(0) + c.arrayToBitmap() + return c.bitmap +} func TestDifferenceRunBitmap(t *testing.T) { a := &container{} @@ -1524,14 +1530,24 @@ func TestDifferenceRunBitmap(t *testing.T) { bitmap: MakeBitmap([]uint64{0x0, 0x8000000000000000}), exp: []interval16{{start: 0, last: 65}}, }, + { + runs: []interval16{{start: 1, last: 65535}}, + bitmap: MakeBitmap([]uint64{0x0000000000000001}), + exp: []interval16{{start: 1, last: 65535}}, + }, + { + runs: []interval16{{start: 0, last: 65533}, {start: 65535, last: 65535}}, + bitmap: MakeLastBitSet(), + exp: []interval16{{start: 0, last: 65533}}, + }, } for i, test := range tests { a.runs = test.runs - a.n = a.runCountRange(0, 100) + a.n = a.runCountRange(0, 65536) for i, v := range test.bitmap { b.bitmap[i] = v } - b.n = b.bitmapCountRange(0, 100) + b.n = b.bitmapCountRange(0, 65536) ret := differenceRunBitmap(a, b) if !reflect.DeepEqual(ret.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) diff --git a/server.go b/server.go index 37270b0f1..bbb02e7c3 100644 --- a/server.go +++ b/server.go @@ -56,6 +56,7 @@ type Server struct { Handler *Handler Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver + RemoteClient *http.Client // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -152,10 +153,10 @@ func (s *Server) Open() error { s.Holder.Peek() // Create default HTTP client - s.createDefaultClient() + s.createDefaultClient(s.RemoteClient) // Create executor for executing queries. - e := NewExecutor(&ClientOptions{TLS: s.TLS}) + e := NewExecutor(s.RemoteClient) e.Holder = s.Holder e.URI = s.URI e.Cluster = s.Cluster @@ -242,6 +243,25 @@ func (s *Server) Addr() net.Addr { } return s.ln.Addr() } +func GetHTTPClient(t *tls.Config) *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 200, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + if t != nil { + transport.TLSClientConfig = t + } + return &http.Client{Transport: transport} +} // Logger returns a logger that writes to LogOutput func (s *Server) Logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } @@ -269,7 +289,7 @@ func (s *Server) monitorAntiEntropy() { syncer.URI = s.URI syncer.Cluster = s.Cluster syncer.Closing = s.closing - syncer.ClientOptions = &ClientOptions{TLS: s.TLS} + syncer.RemoteClient = s.RemoteClient // Sync holders. if err := syncer.SyncHolder(); err != nil { @@ -466,7 +486,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { return nil } -// monitorDiagnostics periodically polls the the Pilosa Indexes for cluster info. +// monitorDiagnostics periodically polls the Pilosa Indexes for cluster info. func (s *Server) monitorDiagnostics() { if s.DiagnosticInterval <= 0 { s.Logger().Printf("diagnostics disabled") @@ -503,7 +523,10 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes())) s.diagnostics.Set("NumFrames", numFrames) s.diagnostics.Set("NumSlices", numSlices) - s.diagnostics.Set("OpenFiles", CountOpenFiles()) + openFiles, err := CountOpenFiles() + if err == nil { + s.diagnostics.Set("OpenFiles", openFiles) + } s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) s.diagnostics.CheckVersion() s.diagnostics.Flush() @@ -553,8 +576,11 @@ func (s *Server) monitorRuntime() { // Record the number of go routines. s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) + openFiles, err := CountOpenFiles() // Open File handles. - s.Holder.Stats.Gauge("OpenFiles", float64(CountOpenFiles()), 1.0) + if err == nil { + s.Holder.Stats.Gauge("OpenFiles", float64(openFiles), 1.0) + } // Runtime memory metrics. runtime.ReadMemStats(&m) @@ -566,35 +592,29 @@ func (s *Server) monitorRuntime() { } } -func (s *Server) createDefaultClient() { - transport := &http.Transport{} - if s.TLS != nil { - transport.TLSClientConfig = s.TLS - } - s.defaultClient = NewInternalHTTPClientFromURI(nil, &ClientOptions{TLS: s.TLS}) +func (s *Server) createDefaultClient(remoteClient *http.Client) { + s.defaultClient = NewInternalHTTPClientFromURI(nil, remoteClient) } // CountOpenFiles on operating systems that support lsof. -func CountOpenFiles() int { - count := 0 - +func CountOpenFiles() (int, error) { switch runtime.GOOS { case "darwin", "linux", "unix", "freebsd": // -b option avoid kernel blocks pid := os.Getpid() out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -b -p %v", pid)).Output() if err != nil { - log.Fatal(err) + return 0, fmt.Errorf("calling lsof: %s", err) } // only count lines with our pid, avoiding warning messages from -b lines := strings.Split(string(out), strconv.Itoa(pid)) - count = len(lines) + return len(lines), nil case "windows": // TODO: count open file handles on windows + return 0, errors.New("CountOpenFiles() on Windows is not supported") default: - + return 0, errors.New("CountOpenFiles() on this OS is not supported") } - return count } // StatusHandler specifies the methods which an object must implement to share diff --git a/server/server.go b/server/server.go index 5237df085..9af9a1947 100644 --- a/server/server.go +++ b/server/server.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "math/rand" "os" "path/filepath" @@ -30,7 +31,6 @@ import ( "time" "crypto/tls" - "io/ioutil" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/gossip" @@ -156,6 +156,7 @@ func (m *Command) SetupServer() error { m.Server.MaxWritesPerRequest = m.Config.MaxWritesPerRequest // Setup TLS + var TLSConfig *tls.Config if uri.Scheme() == "https" { if m.Config.TLS.CertificatePath == "" { return errors.New("certificate path is required for TLS sockets") @@ -171,8 +172,15 @@ func (m *Command) SetupServer() error { Certificates: []tls.Certificate{cert}, InsecureSkipVerify: m.Config.TLS.SkipVerify, } - m.Server.Handler.ClientOptions = &pilosa.ClientOptions{TLS: m.Server.TLS} + + // TODO Review this location + + TLSConfig = m.Server.TLS + } + c := pilosa.GetHTTPClient(TLSConfig) + m.Server.RemoteClient = c + m.Server.Handler.RemoteClient = c // Set the coordinator node. curi, err := pilosa.AddressWithDefaults(m.Config.Cluster.Coordinator) diff --git a/server/server_test.go b/server/server_test.go index 1ca9de2cd..c0837a16a 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -25,7 +25,6 @@ import ( "net" "net/http" "os" - "path/filepath" "reflect" "runtime" "sort" @@ -33,11 +32,9 @@ import ( "strings" "testing" "testing/quick" - "time" "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -53,7 +50,7 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), nil) + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } @@ -326,7 +323,7 @@ func TestMain_FrameRestore(t *testing.T) { defer m2.Close() // Import from first cluster. - client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), nil) + client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } else if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { @@ -363,18 +360,6 @@ func TestConfig_Parse_DataDir(t *testing.T) { } } -// Ensure the "plugins" config can be parsed. -func TestConfig_Parse_Plugins(t *testing.T) { - if c, err := ParseConfig(` -[plugins] -path = "/path/to/plugins" -`); err != nil { - t.Fatal(err) - } else if c.Plugins.Path != "/path/to/plugins" { - t.Fatalf("unexpected path: %s", c.Plugins.Path) - } -} - // tempMkdir makes a temporary directory func tempMkdir(t *testing.T) string { dir, err := ioutil.TempDir("", "pilosatemp") @@ -387,30 +372,19 @@ func tempMkdir(t *testing.T) string { // Ensure the file handle count is working func TestCountOpenFiles(t *testing.T) { // Windows is not supported yet - supported := []string{"darwin", "linux", "unix", "freebsd"} - sort.Strings(supported) - i := sort.Search(len(supported), - func(i int) bool { return supported[i] >= runtime.GOOS }) - if i == len(supported) { - return + if runtime.GOOS == "windows" { + t.Skip("Skipping unsupported CountOpenFiles test on Windows.") } - - // Create directory store temp file - testDir := tempMkdir(t) - defer os.RemoveAll(testDir) - - count := pilosa.CountOpenFiles() - testFile := filepath.Join(testDir, "test.txt") - _, err := os.Create(testFile) + count, err := pilosa.CountOpenFiles() if err != nil { - t.Fatalf("create test file failed: %s", err) + t.Errorf("CountOpenFiles failed: %s", err) } - - if pilosa.CountOpenFiles() < count+1 { - t.Error("Invalid open file handle count") + if count == 0 { + t.Error("CountOpenFiles returned invalid value 0.") } } +/* TODO: Fix this test. See #951. // Ensure program can send/receive broadcast messages. func TestMain_SendReceiveMessage(t *testing.T) { m0 := MustRunMain() @@ -601,6 +575,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { t.Fatal("frame not found") } } +*/ // availablePorts returns a slice of ports that can be used for testing. func availablePorts(cnt int) ([]string, error) { @@ -706,7 +681,7 @@ func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } // Client returns a client to connect to the program. func (m *Main) Client() *pilosa.InternalHTTPClient { - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), nil) + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) if err != nil { panic(err) } diff --git a/test/client.go b/test/client.go index 4ea2fbbad..5eac8cb79 100644 --- a/test/client.go +++ b/test/client.go @@ -1,6 +1,8 @@ package test import ( + "net/http" + "github.com/pilosa/pilosa" ) @@ -10,8 +12,8 @@ type Client struct { } // MustNewClient returns a new instance of Client. Panic on error. -func MustNewClient(host string) *Client { - c, err := pilosa.NewInternalHTTPClient(host, nil) +func MustNewClient(host string, h *http.Client) *Client { + c, err := pilosa.NewInternalHTTPClient(host, h) if err != nil { panic(err) } diff --git a/test/executor.go b/test/executor.go index aedbc8323..5d6872626 100644 --- a/test/executor.go +++ b/test/executor.go @@ -1,6 +1,7 @@ package test import ( + "net/http" "strings" "github.com/pilosa/pilosa" @@ -12,10 +13,16 @@ type Executor struct { *pilosa.Executor } +var remoteClient *http.Client + +func init() { + remoteClient = pilosa.GetHTTPClient(nil) +} + // NewExecutor returns a new instance of Executor. // The executor always matches the uri of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - executor := pilosa.NewExecutor(nil) + executor := pilosa.NewExecutor(remoteClient) e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster