From 6a5aabee51143aadbfc72cbddfe5f01c24daefc9 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 6 Nov 2017 09:24:55 -0600 Subject: [PATCH 01/37] Add docker-build make target for repeatable Docker-based builds. --- Makefile | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 33f1c579d..7563349f6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz +.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz clean docker-build DEP := $(shell command -v dep 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) @@ -12,6 +12,9 @@ LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pil default: test pilosa +clean: + rm -rf vendor build + $(GOPATH)/bin: mkdir $(GOPATH)/bin @@ -51,15 +54,19 @@ pilosa: vendor go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa release-build: vendor +ifdef DOCKER_BUILD + make docker-build FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" +else make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" +endif cp LICENSE README.md build/pilosa-$(IDENTIFIER) tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/ @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz" release: - make release-build GOOS=linux GOARCH=amd64 - make release-build GOOS=linux GOARCH=386 make release-build GOOS=darwin GOARCH=amd64 + make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 + make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 prerelease-build: vendor make pilosa FLAGS="-o build/pilosa-master-$(GOOS)-$(GOARCH)/pilosa" @@ -99,3 +106,6 @@ endif docker: docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) . @echo "Created image: pilosa:$(VERSION)" + +docker-build: + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:latest go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa From f1d87fd3cc2f3f46c953b67573f4fad3215c2ff1 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 6 Nov 2017 10:23:07 -0600 Subject: [PATCH 02/37] bump MaxIdleConns(PerHost) like v0.7 This helps "connection reset" issues when the cluster is under high query load from many clients. This also brings in the default http transport options which were (mistakenly?) removed by the TLS work. I'm not sure what the ramifications would be of using a blank http.Transport{} instead of http.DefaultTransport, but it seems best to avoid a sweeping change. --- client.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/client.go b/client.go index 0a893ddf9..666447586 100644 --- a/client.go +++ b/client.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "log" "math/rand" + "net" "net/http" "net/url" "sort" @@ -70,7 +71,19 @@ func NewInternalHTTPClientFromURI(defaultURI *URI, options *ClientOptions) *Inte if options == nil { options = &ClientOptions{} } - transport := &http.Transport{} + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 200, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } if options.TLS != nil { transport.TLSClientConfig = options.TLS } From ed323a3274992fd4d5b6dd7e71d8e622759206ad Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 7 Nov 2017 14:31:43 -0600 Subject: [PATCH 03/37] Parameterize golang version for docker builds, add docker-test target for running tests within docker --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7563349f6..9987810d3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz clean docker-build +.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz clean docker-build docker-test DEP := $(shell command -v dep 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) @@ -9,6 +9,7 @@ CLONE_URL=github.com/pilosa/pilosa PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) BUILD_TIME=`date -u +%FT%T%z` LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" +DOCKER_GOLANG_IMAGE=golang:latest default: test pilosa @@ -108,4 +109,7 @@ docker: @echo "Created image: pilosa:$(VERSION)" docker-build: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:latest go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) $(DOCKER_GOLANG_IMAGE) go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + +docker-test: + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) $(DOCKER_GOLANG_IMAGE) go test $(TESTFLAGS) $(PKGS) From 35bcc45a799f4a0ecb4f75f15211edc42b1d1d83 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 7 Nov 2017 15:20:22 -0600 Subject: [PATCH 04/37] Add documentation on importing field values. Fixes #924. --- docs/administration.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/administration.md b/docs/administration.md index 23b5d17f0..22db9c797 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -55,6 +55,18 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` +##### Importing Field Values + +If you are using [BSI Range-Encoding](../data-model/#bsi-range-encoding) field values, you can import field values for a single frame and single field using `--field`. The CSV file should be in the format `ColumnID,Value`. + +``` +pilosa import -i project -f stargazer --field star_count project-stargazer-counts.csv +``` + +
+

Note that you must first create a frame with Range Encoding enabled and a field. View Create Frame for more details.

+
+ #### Exporting Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format RowID,ColumnID and sorted by column ID. From 5594736c718916a57f78851471e8b628875d504c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 8 Nov 2017 11:31:46 -0600 Subject: [PATCH 05/37] Skip permissions test when run as root. Fixes #940 --- holder_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/holder_test.go b/holder_test.go index 104bd1bb0..7020a1b8c 100644 --- a/holder_test.go +++ b/holder_test.go @@ -45,6 +45,9 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrIndexPermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() @@ -95,6 +98,9 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrFramePermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() @@ -151,6 +157,9 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrViewPermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() @@ -172,6 +181,9 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrViewFragmentsMkdir", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() @@ -194,6 +206,9 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrFragmentStoragePermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() @@ -240,6 +255,9 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrFragmentCachePermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } h := test.MustOpenHolder() defer h.Close() From 8c0d1ece5517eb0ec374744a579273f4107c6d94 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 8 Nov 2017 11:46:17 -0600 Subject: [PATCH 06/37] Remove "plugins.path" configuration option According to Todd, the current plugin direction is static and there is no need for dynamic loading, so this option is no longer needed. --- cmd/server_test.go | 3 --- config.go | 4 ---- ctl/generate_config.go | 3 --- ctl/server.go | 1 - server/server_test.go | 12 ------------ 5 files changed, 23 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index 53841a5d9..7b983b5e3 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -80,13 +80,10 @@ func TestServerConfig(t *testing.T) { hosts = [ "localhost:19444", ] - [plugins] - path = "/var/sloth" `, validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"}) - v.Check(cmd.Server.Config.Plugins.Path, "/var/sloth") v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9)) return v.Error() }, diff --git a/config.go b/config.go index d68158f5e..cd978c5ca 100644 --- a/config.go +++ b/config.go @@ -81,10 +81,6 @@ type Config struct { LongQueryTime Duration `toml:"long-query-time"` } `toml:"cluster"` - Plugins struct { - Path string `toml:"path"` - } `toml:"plugins"` - AntiEntropy struct { Interval Duration `toml:"interval"` } `toml:"anti-entropy"` diff --git a/ctl/generate_config.go b/ctl/generate_config.go index 87f866bb8..5ddef99b1 100644 --- a/ctl/generate_config.go +++ b/ctl/generate_config.go @@ -60,9 +60,6 @@ max-writes-per-request = 5000 service = "statsd" host = "127.0.0.1:8125" poll-interval = "0m15s" - -[plugins] - path = "" `)+"\n") return nil } diff --git a/ctl/server.go b/ctl/server.go index 2d3b82ca1..c251b8fb0 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -36,7 +36,6 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Long Query Time.") - flags.StringVarP(&srv.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path") flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") diff --git a/server/server_test.go b/server/server_test.go index 548affac9..1b07be671 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -363,18 +363,6 @@ func TestConfig_Parse_DataDir(t *testing.T) { } } -// Ensure the "plugins" config can be parsed. -func TestConfig_Parse_Plugins(t *testing.T) { - if c, err := ParseConfig(` -[plugins] -path = "/path/to/plugins" -`); err != nil { - t.Fatal(err) - } else if c.Plugins.Path != "/path/to/plugins" { - t.Fatalf("unexpected path: %s", c.Plugins.Path) - } -} - // tempMkdir makes a temporary directory func tempMkdir(t *testing.T) string { dir, err := ioutil.TempDir("", "pilosatemp") From e9a2534af7ac914242f25325953ef29d9ba7b7d3 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 8 Nov 2017 13:12:51 -0600 Subject: [PATCH 07/37] Document undocumented flags and add tests (Fixes #915). --- cmd/server_test.go | 6 +++++- ctl/server.go | 2 +- docs/configuration.md | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index 7b983b5e3..04436c1c4 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -45,10 +45,11 @@ func TestServerConfig(t *testing.T) { // TEST 0 { args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"}, - env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_POLL_INTERVAL": "3m2s"}, + env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_POLL_INTERVAL": "3m2s", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, cfgFileContent: ` data-dir = "/tmp/myFileDatadir" bind = "localhost:0" + max-writes-per-request = 3000 [cluster] poll-interval = "45s" @@ -57,6 +58,7 @@ func TestServerConfig(t *testing.T) { hosts = [ "localhost:19444", ] + long-query-time = "1m10s" `, validation: func() error { v := validator{} @@ -65,6 +67,8 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Cluster.ReplicaN, 2) v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"}) v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Second*182)) + v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90)) + v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) return v.Error() }, }, diff --git a/ctl/server.go b/ctl/server.go index c251b8fb0..78f72b732 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -35,7 +35,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? - flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Long Query Time.") + flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path") flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") diff --git a/docs/configuration.md b/docs/configuration.md index 0396e2e95..ba4b7d70a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -76,6 +76,28 @@ Any flag that has a value that is a comma separated list on the command line bec data-dir = "~/.pilosa" ``` +#### Log Path + +* Description: Path of log file +* Flag: `--log-path="/path/to/logfile"` +* Env: `PILOSA_LOG_PATH="/path/to/logfile"` +* Config: + + ```toml + log_path = "/path/to/logfile" + ``` + +#### Max Writes Per Request + +* Description: Maximum number of mutating commands allowed per request. This includes SetBit, ClearBit, SetRowAttrs, SetColumnAttrs, and SetFieldValue. +* Flag: `--max-writes-per-request=5000` +* Env: `PILOSA_MAX_WRITES_PER_REQUEST=5000` +* Config: + + ```toml + max-writes-per-request = 5000 + ``` + #### Gossip Port * Description: Port to which Pilosa should bind for internal communication. @@ -135,6 +157,18 @@ Any flag that has a value that is a comma separated list on the command line bec poll-interval = "1m0s" ``` +#### Cluster Long Query Time + +* Description: Duration that will trigger log and stat messages for slow queries. +* Flag: `cluster.long-query-time="1m0s"` +* Env: `PILOSA_CLUSTER_LONG_QUERY_TIME="1m0s"` +* Config: + + ```toml + [cluster] + long-query-time = "1m0s" + ``` + #### Cluster Replicas * Description: Number of hosts each piece of data should be stored on. From 857b0b3736f36e876570d8d17ab718fc2b730e82 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 8 Nov 2017 14:33:40 -0600 Subject: [PATCH 08/37] End with a period --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index ba4b7d70a..85af6c1a4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -78,7 +78,7 @@ Any flag that has a value that is a comma separated list on the command line bec #### Log Path -* Description: Path of log file +* Description: Path of log file. * Flag: `--log-path="/path/to/logfile"` * Env: `PILOSA_LOG_PATH="/path/to/logfile"` * Config: From 83be24f4d263c1f017524d77ac870a634d5fe7ac Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 10 Nov 2017 18:00:45 +0300 Subject: [PATCH 09/37] Removes column/row labels for input definition. Resolves #810 --- docs/input-definition.md | 6 +----- handler.go | 12 ++++++----- handler_test.go | 42 +++++++-------------------------------- index.go | 3 ++- index_test.go | 4 ++-- input_definition.go | 43 ++++++++++++++++++++++++++++++++++------ input_definition_test.go | 36 +++++++++++---------------------- 7 files changed, 68 insertions(+), 78 deletions(-) diff --git a/docs/input-definition.md b/docs/input-definition.md index da59ee391..f7571b5cc 100644 --- a/docs/input-definition.md +++ b/docs/input-definition.md @@ -43,10 +43,6 @@ curl localhost:10101/index/repository/input-definition/stargazer \ } ], "fields": [ - { - "name": "repo_id", - "primaryKey": true - }, { "actions": [ { @@ -105,8 +101,8 @@ curl localhost:10101/index/repository/input/stargazer \ -X POST \ -d '[ { + "columnID": 91720568, "language_id": "Go", - "repo_id": 91720568, "stargazer_id": 513114, "time_value": "2017-05-18T20:40" }, diff --git a/handler.go b/handler.go index 0f224c31d..f0133a1c3 100644 --- a/handler.go +++ b/handler.go @@ -1760,8 +1760,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - // Validation the input definition with the curent index's ColumnLabel. - if err := req.Validate(index.ColumnLabel()); err != nil { + if err := req.Validate(); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -1908,10 +1907,13 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, for _, field := range inputDef.Fields() { validFields[field.Name] = true if field.PrimaryKey { - columnLabel := field.Name - value, ok := req[columnLabel] + primaryKey := field.Name + if primaryKey != DefaultColumnLabel { + return nil, fmt.Errorf("Primary key field should have the name: %s", DefaultColumnLabel) + } + value, ok := req[primaryKey] if !ok { - return nil, fmt.Errorf("columnLabel required") + return nil, fmt.Errorf("primary key does not exist") } rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float if !ok { diff --git a/handler_test.go b/handler_test.go index fcae35a28..b66cdf830 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1309,34 +1309,6 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { t.Fatalf("unexpected body: %s", body) } - // Eusure throwing error if primary field's name doesn't match columnLabel - hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"}) - unmatchColumnBody := []byte(` - { - "frames":[{ - "name":"event-time", - "options":{ - "timeQuantum": "YMD", - "inverseEnabled": false, - "cacheType": "ranked" - } - }], - "fields": [ - { - "name": "columnID", - "primaryKey": true - } - ] - }`) - - w = httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i1/input-definition/input1", bytes.NewBuffer(unmatchColumnBody))) - if w.Code != http.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionColumnLabel.Error()+"\n" { - t.Fatalf("unexpected body: %s", body) - } - // Eusure throwing error if request body is invalid. jsonErrorBody := []byte(` { @@ -1578,7 +1550,7 @@ func TestHandler_CreateInput(t *testing.T) { } inputBody := []byte(` [{ - "id": 1, + "columnID": 1, "cabType": "yellow", "distanceMiles": 8, "withPet": true, @@ -1659,14 +1631,14 @@ func TestInput_JSON(t *testing.T) { err string }{ {json: `[{ - "id": 1, + "columnID": 1, "cabType": "yellow", "distanceMiles": 8, "nofield": true }]`, err: "field not found: nofield"}, {json: `[{ - "id": "abc", + "columnID": "abc", "cabType": "yellow", "distanceMiles": 8, "withPet": true @@ -1677,23 +1649,23 @@ func TestInput_JSON(t *testing.T) { "distanceMiles": 8, "withPet": true }]`, - err: "columnLabel required"}, + err: "primary key does not exist"}, {json: `[{ - "id": 1, + "columnID": 1, "cabType": "yellow", "distanceMiles": 8, "withPet": true }`, err: "unexpected EOF"}, {json: `[{ - "id": 1, + "columnID": 1, "cabType": "yellow", "distanceMiles": 8, "noFrame": 1 }]`, err: "Frame not found: foo"}, {json: `[{ - "id": 1, + "columnID": 1, "cabType": "yellow", "distanceMiles": 8, "time_value": 12345 diff --git a/index.go b/index.go index 2031ec5fe..bdf2832b0 100644 --- a/index.go +++ b/index.go @@ -687,7 +687,8 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin for _, fr := range pb.Frames { opt := FrameOptions{ - RowLabel: fr.Meta.RowLabel, + // Deprecating row labels per #810. So, setting the default row label here. + RowLabel: DefaultRowLabel, InverseEnabled: fr.Meta.InverseEnabled, CacheType: fr.Meta.CacheType, CacheSize: fr.Meta.CacheSize, diff --git a/index_test.go b/index_test.go index a7cff68f7..dbfc939b5 100644 --- a/index_test.go +++ b/index_test.go @@ -309,14 +309,14 @@ func TestIndex_CreateInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} + fields := internal.InputDefinitionField{PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } else if inputDef.Frames()[0].Name != frames.Name { t.Fatalf("unexpected input definition frames %v", inputDef.Frames()) - } else if inputDef.Fields()[0].Name != fields.Name { + } else if inputDef.Fields()[0].Name != pilosa.DefaultColumnLabel { t.Fatalf("unexpected input definition actions %v", inputDef.Fields()) } } diff --git a/input_definition.go b/input_definition.go index 13b27bd17..3bb55ba79 100644 --- a/input_definition.go +++ b/input_definition.go @@ -90,7 +90,8 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { inputFrame := InputFrame{ Name: fr.Name, Options: FrameOptions{ - RowLabel: frameMeta.RowLabel, + // Deprecating row labels per #810. So, setting the default row label here. + RowLabel: DefaultRowLabel, InverseEnabled: frameMeta.InverseEnabled, CacheSize: frameMeta.CacheSize, CacheType: frameMeta.CacheType, @@ -100,8 +101,11 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { i.frames = append(i.frames, inputFrame) } + primaryKeyGiven := false + for _, field := range pb.Fields { var actions []Action + fieldName := field.Name for _, action := range field.InputDefinitionActions { actions = append(actions, Action{ Frame: action.Frame, @@ -111,14 +115,27 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { }) } + if field.PrimaryKey { + // Deprecating column labels per #810. + // So, setting the default column label here. + fieldName = DefaultColumnLabel + primaryKeyGiven = true + } + inputField := InputDefinitionField{ - Name: field.Name, + Name: fieldName, PrimaryKey: field.PrimaryKey, Actions: actions, } i.fields = append(i.fields, inputField) } + if len(pb.Fields) > 0 && !primaryKeyGiven { + // primary field is required if there are other fields. + // add it if it doesn't exist. + i.fields = append(i.fields, InputDefDefaultPrimaryKeyField()) + } + return nil } @@ -265,7 +282,7 @@ type InputDefinitionInfo struct { } // Validate the InputDefinitionInfo data. -func (i *InputDefinitionInfo) Validate(columnLabel string) error { +func (i *InputDefinitionInfo) Validate() error { numPrimaryKey := 0 accountRowID := make(map[string]uint64) @@ -298,9 +315,6 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error { } if field.PrimaryKey { numPrimaryKey++ - if field.Name != columnLabel { - return ErrInputDefinitionColumnLabel - } } else if len(field.Actions) == 0 { return ErrInputDefinitionActionRequired } @@ -321,7 +335,16 @@ func (i *InputDefinitionInfo) Encode() *internal.InputDefinition { for _, f := range i.Frames { def.Frames = append(def.Frames, f.Encode()) } + primaryKeyGiven := false for _, f := range i.Fields { + if f.PrimaryKey { + f.Name = DefaultColumnLabel + primaryKeyGiven = true + } + def.Fields = append(def.Fields, f.Encode()) + } + if len(i.Fields) > 0 && !primaryKeyGiven { + f := InputDefDefaultPrimaryKeyField() def.Fields = append(def.Fields, f.Encode()) } return &def @@ -379,3 +402,11 @@ func HandleAction(a Action, value interface{}, colID uint64, timestamp int64) (* } return &bit, err } + +func InputDefDefaultPrimaryKeyField() InputDefinitionField { + return InputDefinitionField{ + Name: DefaultColumnLabel, + PrimaryKey: true, + Actions: []Action{}, + } +} diff --git a/input_definition_test.go b/input_definition_test.go index f2daade2c..1912043f0 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -63,10 +63,6 @@ func TestInputDefinition_Encoding(t *testing.T) { } }], "fields": [ - { - "name": "id", - "primaryKey": true - }, { "name": "cabType", "actions": [ @@ -96,9 +92,9 @@ func TestInputDefinition_Encoding(t *testing.T) { t.Fatalf("unexpected frame meta data: %v", internalDef) } else if len(internalDef.Fields) != 2 { t.Fatalf("unexpected number of Fields: %d", len(internalDef.Fields)) - } else if len(internalDef.Fields[1].InputDefinitionActions) != 1 { + } else if len(internalDef.Fields[0].InputDefinitionActions) != 1 { t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].InputDefinitionActions) - } else if internalDef.Fields[1].InputDefinitionActions[0].ValueDestination != "mapping" { + } else if internalDef.Fields[0].InputDefinitionActions[0].ValueDestination != "mapping" { t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].InputDefinitionActions[0]) } } @@ -110,14 +106,14 @@ func TestActionValidation(t *testing.T) { action := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} field := pilosa.InputDefinitionField{Name: "id", PrimaryKey: false, Actions: []pilosa.Action{action}} info := pilosa.InputDefinitionInfo{Fields: []pilosa.InputDefinitionField{field}} - err := info.Validate("id") + err := info.Validate() if err != pilosa.ErrInputDefinitionAttrsRequired { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionAttrsRequired, err) } frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } @@ -126,7 +122,7 @@ func TestActionValidation(t *testing.T) { action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if err != pilosa.ErrName { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrName, err) } @@ -135,23 +131,15 @@ func TestActionValidation(t *testing.T) { action = pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if err != pilosa.ErrFrameRequired { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrFrameRequired, err) } - action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} - field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("test") - if err != pilosa.ErrInputDefinitionColumnLabel { - t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionColumnLabel, err) - } - action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "x", PrimaryKey: false, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if err != pilosa.ErrInputDefinitionHasPrimaryKey { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionHasPrimaryKey, err) } @@ -159,7 +147,7 @@ func TestActionValidation(t *testing.T) { action = pilosa.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if !strings.Contains(err.Error(), "invalid ValueDestination") { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } @@ -167,7 +155,7 @@ func TestActionValidation(t *testing.T) { action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} - err = info.Validate("id") + err = info.Validate() if err != pilosa.ErrInputDefinitionValueMap { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionValueMap, err) } @@ -177,15 +165,15 @@ func TestActionValidation(t *testing.T) { action1 := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field1 := pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action1}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field, field1}} - err = info.Validate("id") + err = info.Validate() if !strings.Contains(err.Error(), "duplicate rowID with other field") { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true} + field = pilosa.InputDefinitionField{PrimaryKey: true} field1 = pilosa.InputDefinitionField{Name: "test", PrimaryKey: false} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field, field1}} - err = info.Validate("id") + err = info.Validate() if err != pilosa.ErrInputDefinitionActionRequired { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionActionRequired, err) } From b701fb68b0733670ba63c6475e82eabcff531d5f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 8 Nov 2017 13:30:02 -0600 Subject: [PATCH 10/37] Remove unneeded Gopkg.toml constraints and update all dependencies. Fixes #926. --- Gopkg.lock | 42 ++++++++++++++++++++++-------------------- Gopkg.toml | 52 +++------------------------------------------------- 2 files changed, 25 insertions(+), 69 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 09793a7e2..ed772ef39 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -4,7 +4,8 @@ [[projects]] name = "github.com/BurntSushi/toml" packages = ["."] - revision = "99064174e013895bbd9b025c31100bd1d9b590ca" + revision = "b26d9c308763d68093482582cea63d69be07a0f0" + version = "v0.3.0" [[projects]] branch = "master" @@ -27,7 +28,8 @@ [[projects]] name = "github.com/boltdb/bolt" packages = ["."] - revision = "4b1ebc1869ad66568b313d0dc410e2be72670dda" + revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" + version = "v1.3.1" [[projects]] name = "github.com/davecgh/go-spew" @@ -51,13 +53,13 @@ branch = "master" name = "github.com/golang/groupcache" packages = ["lru"] - revision = "b710c8433bd175204919eb38776e944233235d03" + revision = "84a468cf14b4376def5d68c722b139b881c450a4" [[projects]] branch = "master" name = "github.com/golang/protobuf" packages = ["proto"] - revision = "130e6b02ab059e7b717a096f397c5b60111cae74" + revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9" [[projects]] name = "github.com/gorilla/context" @@ -68,8 +70,8 @@ [[projects]] name = "github.com/gorilla/mux" packages = ["."] - revision = "24fca303ac6da784b9e8269f724ddeb0b2eea5e7" - version = "v1.5.0" + revision = "7f08801859139f86dfafd1c296e2cba9a80d292e" + version = "v1.6.0" [[projects]] branch = "master" @@ -99,7 +101,7 @@ branch = "master" name = "github.com/hashicorp/go-sockaddr" packages = ["."] - revision = "41949a141473f6340abc6ba0fcd0f89da6f6f837" + revision = "9b4c5fa5b10a683339a270d664474b9f4aee62fc" [[projects]] branch = "master" @@ -111,7 +113,7 @@ branch = "master" name = "github.com/hashicorp/hcl" packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] - revision = "68e816d1c783414e79bc65b3994d9ab6b0a722ab" + revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" [[projects]] name = "github.com/hashicorp/memberlist" @@ -135,13 +137,13 @@ branch = "master" name = "github.com/miekg/dns" packages = [".","internal/socket"] - revision = "946bd9fbed05568b0f3cd188353d8aa28f38b688" + revision = "9fc4eb252eedf0ef8adc05169ce35da5e31beaba" [[projects]] branch = "master" name = "github.com/mitchellh/mapstructure" packages = ["."] - revision = "d0303fe809921458f417bcf828397a65db30a7e4" + revision = "06020f85339e21b2478f756a78e295255ffa4d6a" [[projects]] name = "github.com/pelletier/go-toml" @@ -171,7 +173,7 @@ branch = "master" name = "github.com/spf13/afero" packages = [".","mem"] - revision = "e67d870304c4bca21331b02f414f970df13aa694" + revision = "5660eeed305fe5f69c8fc6cf899132a459a97064" [[projects]] name = "github.com/spf13/cast" @@ -180,10 +182,10 @@ version = "v1.1.0" [[projects]] - branch = "master" name = "github.com/spf13/cobra" packages = ["."] - revision = "50204810fdb5010baae72e4f41b303689cbdcc9f" + revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" + version = "v0.0.1" [[projects]] branch = "master" @@ -198,34 +200,34 @@ version = "v1.0.0" [[projects]] - branch = "master" name = "github.com/spf13/viper" packages = ["."] - revision = "d9cca5ef33035202efb1586825bdbb15ff9ec3ba" + revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" + version = "v1.0.0" [[projects]] branch = "master" name = "golang.org/x/net" packages = ["context"] - revision = "a04bdaca5b32abe1c069418fb7088ae607de5bd0" + revision = "a337091b0525af65de94df2eb7e98bd9962dcbe2" [[projects]] branch = "master" name = "golang.org/x/sync" packages = ["errgroup"] - revision = "8e0aa688b654ef28caa72506fa5ec8dba9fc7690" + revision = "fd80eb99c8f653c847d294a001bdf2a3a6f768f5" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "ebfc5b4631820b793c9010c87fd8fef0f39eb082" + revision = "1e2299c37cc91a509f1b12369872d27be0ce98a6" [[projects]] branch = "master" name = "golang.org/x/text" packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"] - revision = "825fc78a2fd6fa0a5447e300189e3219e05e1f25" + revision = "88f656faf3f37f690df1a32515b479415e1a6769" [[projects]] branch = "v2" @@ -236,6 +238,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "0a7eaef315413bc55acf42c5566982fb71be8c11f33162127953572738967b35" + inputs-digest = "75badb0bcc3bb356b04af17979e0af61b4b66c5e0a483f09e39cf1f9b5e5de2c" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index a91539345..7ffa67a2f 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,49 +1,3 @@ - -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - -[[constraint]] - name = "github.com/BurntSushi/toml" - revision = "99064174e013895bbd9b025c31100bd1d9b590ca" - -[[constraint]] - branch = "master" - name = "github.com/CAFxX/gcnotifier" - -[[constraint]] - name = "github.com/boltdb/bolt" - version = "1.3.0" - -[[constraint]] - name = "github.com/gorilla/mux" - version = "1.3.0" - -[[constraint]] - name = "github.com/hashicorp/memberlist" - version = "=0.1.0" - -[[constraint]] - branch = "master" - name = "github.com/spf13/cobra" - -[[constraint]] - branch = "master" - name = "github.com/spf13/viper" +# This file intentionally left blank as all needed dependencies are imported by +# the project and thus tracked by `dep`. +# See https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md for details. From b039747887833e811e82d49a5d07cdbbf517fb17 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 10 Nov 2017 10:45:56 -0600 Subject: [PATCH 11/37] fix a few typos --- cache.go | 2 +- diagnostics/diagnostics.go | 2 +- server.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cache.go b/cache.go index 2e66d35b9..a6f37a409 100644 --- a/cache.go +++ b/cache.go @@ -215,7 +215,7 @@ func (c *RankCache) IDs() []uint64 { return a } -// Invalidate recalculates the the entries by rank. +// Invalidate recalculates the entries by rank. func (c *RankCache) Invalidate() { c.mu.Lock() defer c.mu.Unlock() diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index c653a7052..6bcd433d3 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -175,7 +175,7 @@ func (d *Diagnostics) CompareVersion(value string) error { } else if localVersion[1] < currentVersion[1] { // Minor return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value) } else if localVersion[2] < currentVersion[2] { // Patch - return fmt.Errorf("There is a new patch release of Pilosa availbale: %s: https://github.com/pilosa/pilosa/releases", value) + return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value) } return nil diff --git a/server.go b/server.go index 610fe6255..0d55e7c47 100644 --- a/server.go +++ b/server.go @@ -497,7 +497,7 @@ func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint return s.defaultClient.MaxSliceByIndex(ctx) } -// monitorDiagnostics periodically polls the the Pilosa Indexes for cluster info. +// monitorDiagnostics periodically polls the Pilosa Indexes for cluster info. func (s *Server) monitorDiagnostics() { if s.DiagnosticInterval <= 0 { s.Logger().Printf("diagnostics disabled") From 50d31a3c830db47f253d832f0f292c935176eea3 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 10 Nov 2017 12:44:25 -0600 Subject: [PATCH 12/37] fix overflow in differenceRunBitmap --- roaring/roaring.go | 5 ++++- roaring/roaring_internal_test.go | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 62f87d94b..e4d75a0d2 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2508,7 +2508,7 @@ func differenceRunArray(a, b *container) *container { // differenceRunBitmap computes the difference of an run from a bitmap. func differenceRunBitmap(a, b *container) *container { // If a is full, difference is the flip of b. - if a.runs[0].start == 0 && a.runs[0].last == 65535 { + if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 { return b.flipBitmap() } output := &container{container_type: ContainerRun} @@ -2534,6 +2534,9 @@ func differenceRunBitmap(a, b *container) *container { break } } + if bit == 65535 { //overflow + break + } } if run.start <= run.last { output.runs = append(output.runs, run) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index bdc08b649..a4b963a13 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1524,14 +1524,19 @@ func TestDifferenceRunBitmap(t *testing.T) { bitmap: MakeBitmap([]uint64{0x0, 0x8000000000000000}), exp: []interval16{{start: 0, last: 65}}, }, + { + runs: []interval16{{start: 1, last: 65535}}, + bitmap: MakeBitmap([]uint64{0x0000000000000001}), + exp: []interval16{{start: 1, last: 65535}}, + }, } for i, test := range tests { a.runs = test.runs - a.n = a.runCountRange(0, 100) + a.n = a.runCountRange(0, 65535) for i, v := range test.bitmap { b.bitmap[i] = v } - b.n = b.bitmapCountRange(0, 100) + b.n = b.bitmapCountRange(0, 65535) ret := differenceRunBitmap(a, b) if !reflect.DeepEqual(ret.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) From 7a5459da73410e8950ae10cfa2c9ca19ee43a4c7 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Sat, 11 Nov 2017 00:49:37 +0300 Subject: [PATCH 13/37] input defs: Throw an error if pirmary key is not given instead of adding it auto. --- docs/input-definition.md | 3 +++ handler_test.go | 37 +++++++++++++++++++++++++++++++++++++ input_definition.go | 21 +-------------------- input_definition_test.go | 7 +++++-- 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/docs/input-definition.md b/docs/input-definition.md index f7571b5cc..381388198 100644 --- a/docs/input-definition.md +++ b/docs/input-definition.md @@ -43,6 +43,9 @@ curl localhost:10101/index/repository/input-definition/stargazer \ } ], "fields": [ + { + "primaryKey": true + }, { "actions": [ { diff --git a/handler_test.go b/handler_test.go index b66cdf830..280467cfc 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1309,6 +1309,43 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { t.Fatalf("unexpected body: %s", body) } + // Ensure throwing error if there's no primary key + hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"}) + unmatchColumnBody := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": false, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "foo", + "actions": [ + { + "frame": "cab-type", + "valueDestination": "mapping", + "valueMap": { + "Green": 1, + "Yellow": 2 + } + } + ] + } + ] + }`) + + w = httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i1/input-definition/input1", bytes.NewBuffer(unmatchColumnBody))) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionHasPrimaryKey.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + // Eusure throwing error if request body is invalid. jsonErrorBody := []byte(` { diff --git a/input_definition.go b/input_definition.go index 3bb55ba79..0c5088614 100644 --- a/input_definition.go +++ b/input_definition.go @@ -131,9 +131,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { } if len(pb.Fields) > 0 && !primaryKeyGiven { - // primary field is required if there are other fields. - // add it if it doesn't exist. - i.fields = append(i.fields, InputDefDefaultPrimaryKeyField()) + return ErrInputDefinitionHasPrimaryKey } return nil @@ -335,16 +333,7 @@ func (i *InputDefinitionInfo) Encode() *internal.InputDefinition { for _, f := range i.Frames { def.Frames = append(def.Frames, f.Encode()) } - primaryKeyGiven := false for _, f := range i.Fields { - if f.PrimaryKey { - f.Name = DefaultColumnLabel - primaryKeyGiven = true - } - def.Fields = append(def.Fields, f.Encode()) - } - if len(i.Fields) > 0 && !primaryKeyGiven { - f := InputDefDefaultPrimaryKeyField() def.Fields = append(def.Fields, f.Encode()) } return &def @@ -402,11 +391,3 @@ func HandleAction(a Action, value interface{}, colID uint64, timestamp int64) (* } return &bit, err } - -func InputDefDefaultPrimaryKeyField() InputDefinitionField { - return InputDefinitionField{ - Name: DefaultColumnLabel, - PrimaryKey: true, - Actions: []Action{}, - } -} diff --git a/input_definition_test.go b/input_definition_test.go index 1912043f0..4daf7ce89 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -63,6 +63,9 @@ func TestInputDefinition_Encoding(t *testing.T) { } }], "fields": [ + { + "primaryKey": true + }, { "name": "cabType", "actions": [ @@ -92,9 +95,9 @@ func TestInputDefinition_Encoding(t *testing.T) { t.Fatalf("unexpected frame meta data: %v", internalDef) } else if len(internalDef.Fields) != 2 { t.Fatalf("unexpected number of Fields: %d", len(internalDef.Fields)) - } else if len(internalDef.Fields[0].InputDefinitionActions) != 1 { + } else if len(internalDef.Fields[1].InputDefinitionActions) != 1 { t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].InputDefinitionActions) - } else if internalDef.Fields[0].InputDefinitionActions[0].ValueDestination != "mapping" { + } else if internalDef.Fields[1].InputDefinitionActions[0].ValueDestination != "mapping" { t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].InputDefinitionActions[0]) } } From 450dda7fd0760d54580c73a07d16d4ca4078b6ba Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 10 Nov 2017 17:01:25 -0600 Subject: [PATCH 14/37] overflow bug in differenceRunBitmap Part 2 --- roaring/roaring.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index e4d75a0d2..97994c37a 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2513,12 +2513,20 @@ func differenceRunBitmap(a, b *container) *container { } output := &container{container_type: ContainerRun} output.n = a.n + if len(a.runs) == 0 { + return output + } for j := 0; j < len(a.runs); j++ { run := a.runs[j] + add := true for bit := a.runs[j].start; bit <= a.runs[j].last; bit++ { if b.bitmapContains(bit) { output.n-- if run.start == bit { + if bit == 65535 { //overflow + add = false + } + run.start++ } else if bit == run.last { run.last-- @@ -2534,13 +2542,15 @@ func differenceRunBitmap(a, b *container) *container { break } } + if bit == 65535 { //overflow break } } if run.start <= run.last { - output.runs = append(output.runs, run) - + if add { + output.runs = append(output.runs, run) + } } } From c9c2a2b0d838a9aaa37951244b972ad0337bd3a6 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Sat, 11 Nov 2017 09:44:15 -0600 Subject: [PATCH 15/37] added test for differenceRunBitmap overflow bug --- roaring/roaring_internal_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index a4b963a13..b36723b26 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1485,6 +1485,12 @@ func MakeBitmap(start []uint64) []uint64 { } return b } +func MakeLastBitSet() []uint64 { + obj := NewBitmap(65535) + c := obj.container(0) + c.arrayToBitmap() + return c.bitmap +} func TestDifferenceRunBitmap(t *testing.T) { a := &container{} @@ -1529,14 +1535,19 @@ func TestDifferenceRunBitmap(t *testing.T) { bitmap: MakeBitmap([]uint64{0x0000000000000001}), exp: []interval16{{start: 1, last: 65535}}, }, + { + runs: []interval16{{start: 0, last: 65533}, {start: 65535, last: 65535}}, + bitmap: MakeLastBitSet(), + exp: []interval16{{start: 0, last: 65533}}, + }, } for i, test := range tests { a.runs = test.runs - a.n = a.runCountRange(0, 65535) + a.n = a.runCountRange(0, 65536) for i, v := range test.bitmap { b.bitmap[i] = v } - b.n = b.bitmapCountRange(0, 65535) + b.n = b.bitmapCountRange(0, 65536) ret := differenceRunBitmap(a, b) if !reflect.DeepEqual(ret.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) From 1a965e7f9ccee9930704335963640bd07479cdd9 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 13 Nov 2017 15:55:01 -0600 Subject: [PATCH 16/37] comment out test TestMain_SendReceiveMessage until addressing several issues --- server/server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/server_test.go b/server/server_test.go index 1b07be671..849f12b4b 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -33,11 +33,9 @@ import ( "strings" "testing" "testing/quick" - "time" "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -399,6 +397,7 @@ func TestCountOpenFiles(t *testing.T) { } } +/* TODO: Fix this test. See #951. // Ensure program can send/receive broadcast messages. func TestMain_SendReceiveMessage(t *testing.T) { m0 := MustRunMain() @@ -580,6 +579,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { t.Fatal("frame not found") } } +*/ // availablePorts returns a slice of ports that can be used for testing. func availablePorts(cnt int) ([]string, error) { From cf85a08596559cb4691e314fb45906c540c2e0a7 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 13 Nov 2017 18:15:15 -0600 Subject: [PATCH 17/37] Add search-friendly documentation for BSI range query syntax --- docs/query-language.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 2a8130736..8d39f818a 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -427,7 +427,7 @@ Returns `{{"attrs":{},"bits":[10]}` **Spec:** ``` -Range(, ) +Range(, ) ``` **Description:** @@ -441,7 +441,7 @@ Returns bits that are true for the comparison operator. **Examples:** In our source data, commitactivity was counted over the last year. -The following Range query returns all repositories having more than 100 commits. +The following greater-than (GT) Range query returns all repositories having more than 100 commits. ``` Range(frame="stats", commitactivity > 100) @@ -451,6 +451,13 @@ Returns `{{"attrs":{},"bits":[10]}` * bits are repositories which had at least 100 commits in the last year. +Similar syntax is supported for less-than (LT or `<`), less-than-or-equal (LTE or `<=`), and greater-than-or-equal (GTE or `>=`). An interval with both bounds can be specified with the "BETWEEN" operator `><`, and a two-element list containing the lower and upper bounds of the interval: + +``` +Range(frame="stats", commitactivity >< [100, 200]) +``` + +This is conceptually equivalent to the interval 100 < commitactivity < 200, but this chained comparison syntax is not currently supported. BETWEEN queries syntax is restricted to greater-than and less-than, but any valid interval on the integers can be represented this way. #### Sum @@ -499,4 +506,4 @@ SetFieldValue returns `null` upon success. Set the number of pull requests of repository 10. ``` SetFieldValue(col=10, frame="stats", pullrequests=2) -``` \ No newline at end of file +``` From 7a02bfe6f293d3f3208007a14c863d1191aa075c Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 14 Nov 2017 09:59:39 -0600 Subject: [PATCH 18/37] Format list properly --- docs/administration.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 22db9c797..bc40fa41c 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -128,21 +128,21 @@ Note: This will only work when the replication factor is >= 2 Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions. -Version: Version string of the build. -Host: Host URI. -Cluster: List of nodes in the Cluster. -NumNodes: Number of nodes in the Cluster. -NumCPU: Number of Cores per Node -BSIEnabled: Bit Slice Index Frames in use. -TimeQuantumEnabled: Time Quantum Frames in use. -InverseEnabled: Inverse Frames in use. -NumIndexes: Number of Indexes in the Cluster. -NumFrames: Number of Frames in the Cluster. -NumSlices: Number of Slices in the Cluster. -NumViews: Number of Views in the Cluster. -OpenFiles: Open file handle count. -GoRoutines: Go routine count. - +- Version: Version string of the build. +- Host: Host URI. +- Cluster: List of nodes in the Cluster. +- NumNodes: Number of nodes in the Cluster. +- NumCPU: Number of Cores per Node +- BSIEnabled: Bit Slice Index Frames in use. +- TimeQuantumEnabled: Time Quantum Frames in use. +- InverseEnabled: Inverse Frames in use. +- NumIndexes: Number of Indexes in the Cluster. +- NumFrames: Number of Frames in the Cluster. +- NumSlices: Number of Slices in the Cluster. +- NumViews: Number of Views in the Cluster. +- OpenFiles: Open file handle count. +- GoRoutines: Go routine count. + You can opt-out of the Pilosa diagnostics reporting by setting either the command line configuration option `--metric.diagnostics=false`, use the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option. #### Metrics From f756e9eee6554cff31a442b8236d080eeb50c343 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 14 Nov 2017 19:15:17 +0300 Subject: [PATCH 19/37] Updates --- handler.go | 6 +----- input_definition.go | 9 ++++----- input_definition_test.go | 3 ++- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/handler.go b/handler.go index f0133a1c3..f3d9fe9d3 100644 --- a/handler.go +++ b/handler.go @@ -1907,11 +1907,7 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, for _, field := range inputDef.Fields() { validFields[field.Name] = true if field.PrimaryKey { - primaryKey := field.Name - if primaryKey != DefaultColumnLabel { - return nil, fmt.Errorf("Primary key field should have the name: %s", DefaultColumnLabel) - } - value, ok := req[primaryKey] + value, ok := req[field.Name] if !ok { return nil, fmt.Errorf("primary key does not exist") } diff --git a/input_definition.go b/input_definition.go index 0c5088614..84d22d4ac 100644 --- a/input_definition.go +++ b/input_definition.go @@ -105,7 +105,6 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { for _, field := range pb.Fields { var actions []Action - fieldName := field.Name for _, action := range field.InputDefinitionActions { actions = append(actions, Action{ Frame: action.Frame, @@ -116,14 +115,11 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { } if field.PrimaryKey { - // Deprecating column labels per #810. - // So, setting the default column label here. - fieldName = DefaultColumnLabel primaryKeyGiven = true } inputField := InputDefinitionField{ - Name: fieldName, + Name: field.Name, PrimaryKey: field.PrimaryKey, Actions: actions, } @@ -296,6 +292,9 @@ func (i *InputDefinitionInfo) Validate() error { // Validate columnLabel and duplicate primaryKey. for _, field := range i.Fields { + if field.Name == "" { + return ErrInputDefinitionNameRequired + } for _, action := range field.Actions { if err := action.Validate(); err != nil { return err diff --git a/input_definition_test.go b/input_definition_test.go index 4daf7ce89..93d0e393d 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -64,6 +64,7 @@ func TestInputDefinition_Encoding(t *testing.T) { }], "fields": [ { + "name": "id", "primaryKey": true }, { @@ -173,7 +174,7 @@ func TestActionValidation(t *testing.T) { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - field = pilosa.InputDefinitionField{PrimaryKey: true} + field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true} field1 = pilosa.InputDefinitionField{Name: "test", PrimaryKey: false} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field, field1}} err = info.Validate() From 921c1dd23fd9762961c5c254639a2d2be52b792e Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 14 Nov 2017 19:20:54 +0300 Subject: [PATCH 20/37] fixed doc --- docs/input-definition.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/input-definition.md b/docs/input-definition.md index 381388198..494f4757e 100644 --- a/docs/input-definition.md +++ b/docs/input-definition.md @@ -44,6 +44,7 @@ curl localhost:10101/index/repository/input-definition/stargazer \ ], "fields": [ { + "name": "repo_id", "primaryKey": true }, { @@ -104,7 +105,7 @@ curl localhost:10101/index/repository/input/stargazer \ -X POST \ -d '[ { - "columnID": 91720568, + "repo_id": 91720568, "language_id": "Go", "stargazer_id": 513114, "time_value": "2017-05-18T20:40" From 52c0676d34ed3c01dd8dc8166f30d0dc5332798d Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 14 Nov 2017 10:28:25 -0600 Subject: [PATCH 21/37] BSI Range doc updates --- docs/query-language.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 8d39f818a..f28257fe6 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -441,7 +441,7 @@ Returns bits that are true for the comparison operator. **Examples:** In our source data, commitactivity was counted over the last year. -The following greater-than (GT) Range query returns all repositories having more than 100 commits. +The following greater-than Range query returns all repositories having more than 100 commits. ``` Range(frame="stats", commitactivity > 100) @@ -451,13 +451,25 @@ Returns `{{"attrs":{},"bits":[10]}` * bits are repositories which had at least 100 commits in the last year. -Similar syntax is supported for less-than (LT or `<`), less-than-or-equal (LTE or `<=`), and greater-than-or-equal (GTE or `>=`). An interval with both bounds can be specified with the "BETWEEN" operator `><`, and a two-element list containing the lower and upper bounds of the interval: +BSI range queries support the following operators: + + Operator | Name | Value +----------|-------------------------------|-------------------- + `>` | greater-than, GT | integer + `<` | less-than, LT | integer + `<=` | less-than-or-equal-to, LTE | integer + `>=` | greater-than-or-equal-to, GTE | integer + `==` | equal-to, EQ | integer + `!=` | not-equal-to, NEQ | integer or `null` + `><` | between, BETWEEN | [integer, integer] + +The `BETWEEN` query specifies an interval with both bounds, using `><` operator, and a two-element list containing the lower and upper bounds of the interval: ``` Range(frame="stats", commitactivity >< [100, 200]) ``` -This is conceptually equivalent to the interval 100 < commitactivity < 200, but this chained comparison syntax is not currently supported. BETWEEN queries syntax is restricted to greater-than and less-than, but any valid interval on the integers can be represented this way. +This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way. #### Sum From 617dc2361c38a48b021ca086a8b3a9305895e227 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 14 Nov 2017 11:12:35 -0600 Subject: [PATCH 22/37] Rename tutorials to examples, how-tos to tutorials --- docs/examples.md | 347 ++++++++++++++++++++++++++++++++++ docs/getting-started.md | 2 +- docs/howtos.md | 214 --------------------- docs/tutorials.md | 399 ++++++++++++++-------------------------- 4 files changed, 481 insertions(+), 481 deletions(-) create mode 100644 docs/examples.md delete mode 100644 docs/howtos.md diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 000000000..bf5e76f29 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,347 @@ ++++ +title = "Examples" +weight = 4 +nav = [ + "Transportation", + "Chemical similarity search", +] ++++ + +## Examples + +### Transportation + +#### Introduction + +New York City released an extremely detailed data set of over 1 billion taxi rides taken in the city - this data has become a popular target for analysis by tech bloggers and has been very well studied. For this reason, we thought it would be interesting to import this data to Pilosa in order to compare with other data stores and techniques on the exact same data set. + +Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions). + +We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step. + +After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa. + +#### Data Model + +The NYC taxi data is comprised of a number of csv files listed here: http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml. These data files have around 20 columns, about half of which are relevant to the benchmark queries we're looking at: + +* Distance: miles, floating point +* Fare: dollars, floating point +* Number of passengers: integer +* Dropoff location: latitude and longitude, floating point +* Pickup location: latitude and longitude, floating point +* Dropoff time: timestamp +* Pickup time: timestamp + +We import these fields, creating one or more Pilosa frames from each of them: + +frame |mapping +------------|--------------------- +cab_type |direct map of enum int → row ID +dist_miles |round(dist) → row ID +total_amount_dollars |round(dist) → row ID +passenger_count |direct map of integer value → row ID +drop_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID +drop_year |year(timestamp) → row ID +drop_month |month(timestamp) → row ID +drop_day |day(timestamp) → row ID +drop_time |time of day mapped to one of 48 half-hour buckets +pickup_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID +pickup_year |year(timestamp) → row ID +pickup_month |month(timestamp) → row ID +pickup_day |day(timestamp) → row ID +pickup_time |time of day mapped to one of 48 half-hour buckets → row ID + +We also created two extra frames that represent the duration and average speed of each ride: + +frame |mapping +--------------------|------------- +duration_minutes |round(drop_timestamp - pickup_timestamp) → row ID +speed_mph |round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID + +#### Mapping + +Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities. + +##### 0 columns → 1 frame + +cab_type: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant. + +##### 1 column → 1 frame + +The following three frames are mapped in a simple direct way from single columns of the original data. + +dist_miles: each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times. + +In PDK parlance, we define a Mapper, which is simply a function that returns integer row IDs. PDK has a number of predefined mappers that can be described with a few parameters. One of these is LinearFloatMapper, which applies a linear function to the input, and casts it to an integer, so the rounding is handled implicitly. In code: +```go +lfm := pdk.LinearFloatMapper{ + Min: -0.5, + Max: 3600.5, + Res: 3601, +} +``` + +`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three. + +This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`). +```go +pdk.BitMapper{ + Frame: "dist_miles", + Mapper: lfm, + Parsers: []pdk.Parser{pdk.FloatParser{}}, + Fields: []int{fields["trip_distance"]}, +}, +``` + +These same objects are represented in the JSON definition file: +```go +{ + "Fields": { + "Trip_distance": 10 + }, + "Mappers": [ + { + "Name": "lfm0", + "Min": -0.5, + "Max": 3600.5, + "Res": 3600 + } + ], + "BitMappers": [ + { + "Frame": "dist_miles", + "Mapper": { + "Name": "lfm0" + }, + "Parsers": [ + {"Name": "FloatParser"} + ], + "Fields": "Trip_distance" + } + ] +} +``` + +Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of BitMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names to column indices. We use these names in the BitMapper definitions to keep things human-readable. + +**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The BitMapper definition is very similar to the previous one. + +**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID. + +##### 1 column → multiple frames + +When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis. + +We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day". + +We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day". + +We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time. + +##### Multiple columns → 1 frame + +The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID. + +We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id. + +Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient. + +##### Complex mappings + +We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work: +```go +durm := pdk.CustomMapper{ + Func: func(fields ...interface{}) interface{} { + start := fields[0].(time.Time) + end := fields[1].(time.Time) + return end.Sub(start).Minutes() + }, + Mapper: lfm, +} +``` + +#### Import process + +After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process. + +#### Queries + +Now we can run some example queries. + +Count per cab type can be retrieved, sorted, with a single PQL call. + +``` +TopN(frame=cab_type) +``` + +High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs. + +``` +TopN(frame=pickup_grid_id) +``` + +Average of total_amount per passenger_count can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average. + +```python +queries = '' +pcounts = range(10) +for i in pcounts: + queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i +resp = requests.post(qurl, data=queries) + +average_amounts = [] +for pcount, topn in zip(pcounts, resp.json()['results']): + wsum = sum([r['count'] * r['key'] for r in topn]) + count = sum([r['count'] for r in topn]) + average_amounts.append(float(wsum)/count) +``` + +For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb). + +### Chemical similarity search + +#### Overview + +The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals. + +We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set. + +Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient. +``` +T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B)) +``` + +A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical. + +All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase + +#### Data model + +We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. + +Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions. + +Given a SMILES encoded molecule and a similarity threshold, we want to retrieve all molecule ids (or SMILES) that have a similarity percentage greater than or equal to the similarity threshold. For example, given a molecule with: +``` +SMILES = "IC=C1/CCC(C(=O)O1)c2cccc3ccccc23" +threshold = 90 +``` + +return the set of molecules that have at least a 90% similarity with the given molecule. + +The Inverse view swaps the rows and columns automatically to enable queries over either the chembl_id or fingerprint. + +Standard View is used to calculate similarity +``` +Index: mole + View: Standard + Col: chembl_id + Frame: fingerprint + Row: position_id ("on" bit positions of a fingerprint) +``` + +Inverse View is used for finding chembl_id based on given SMILES. +From a given SMILES, we use RDKit to convert it to fingerprints with "on" bit position. From "on" bit positions, we can search a list of chembl_ids that match the bit positions. To choose the right chembl_id, we need another query to Standard View then choose the right chembl_id which has the length that matches the given fingerprint's length after using RDKit to convert SMILES to fingerprint. +``` +Index: mole + View: Inverse + Col: position_id ("on" bit positions of a fingerprint) + Frame: fingerprint + Row: chembl_id +``` + +After retrieving chembl_id from the Inverse View, we can use the Tanimoto coefficient to compare chembl_id with the entire data set of molecules. The result of this comparison is the list of `chembl_id`s that have a Tanimoto coefficient greater than the given threshold. + +#### Import process + +To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key. + +For the `mole` index, each row in the csv file has the format 'chembl_id, position_id' by running the following command from Chem-usecase: +``` +python import_from_sdf.py -p -file id_fingerprint.csv +``` + + +First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. +The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393 +``` +curl localhost:10101/index/mole \ + -X POST \ + -d '{"options": {"columnLabel": "position_id"}}' + +curl localhost:10101/index/mole/frame/fingerprint \ + -X POST \ + -d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}' + +``` + +Run the following commands to import the csv data into the `mole` index: +``` +pilosa import -d mole -f fingerprint id_fingerprint.csv +``` + +#### Queries + +Get chembl_id from a given SMILES: +``` +python get_mol_fr_smile.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" +``` + +Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get all chemlb_id that have positions are on, which following these steps: + +* Convert SMILES to fingerprint bit "on" positions + + ```python + from rdkit import Chem + from rdkit.Chem import AllChem + mol=Chem.MolFromSmiles("I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23") + fp = list(AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=4096).GetOnBits()) + ``` + +* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id + + ```python + bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] + bitmap_string = ', '.join(bit_maps) + intersection = "Intersect(%s)" % bitmap_string + mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"] + ``` + +* From list of chembl_id, query all "on" position from mol index, if the length of array of "on" position is matched to len(fp) then return that chembl_id, otherwise the given SMILES does not exist. + + ```python + for m in mole_ids: + mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"] + existed_mol = False + if len(mol) == len(fp): + found = m + existed_mol = True + break + ``` + +Retrieve molecule_ids that have similarity with SMILES="I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" and similarity threshold = 70% +``` +python similar.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" -t 70 +``` + +Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN query to get all chemlb_id that have position is on, which following these steps: + +* Get chembl_id from a SMILES (steps discussed above) + +* Query Pilosa’s TopN to get list of similarity chembl_id + ```python + query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)' + topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string) + ``` + +#### Benchmark + +To run benchmark for specific chembl_id for different similarity threshold at percentage of [50, 70, 75, 80, 85, 90], run following command: +``` +python benchmarks.py -id 6223 +``` + +As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa. + +Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster diff --git a/docs/getting-started.md b/docs/getting-started.md index 84c5fd0bf..1bd2fae03 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -155,4 +155,4 @@ curl localhost:10101/index/repository/query \ ### What's Next? -You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Tutorials](../tutorials/) for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/). +You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Examples](../examples/) page for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/). diff --git a/docs/howtos.md b/docs/howtos.md deleted file mode 100644 index 7809b7091..000000000 --- a/docs/howtos.md +++ /dev/null @@ -1,214 +0,0 @@ -+++ -title = "How Tos" -weight = 4 -nav = [ - "how-to-setup-a-secure-pilosa-cluster" -] -+++ - -## How Tos - -### How To Setup a Secure Pilosa Cluster - -#### Introduction - -Pilosa supports encrypting the communication between and to nodes in a cluster using TLS. In this tutorial, we will be setting up a three node Pilosa cluster running on the same computer. The same steps can be used for a multi-computer cluster but that requires setting up firewalls and other platform-specific configuration which is out of the scope of this tutorial. - -This tutorial assumes that you are using a UNIX-like system, such as Linux or MacOS. [Windows Subsystem for Linux (WSL)](https://msdn.microsoft.com/en-us/commandline/wsl/about) works equally well on Windows 10 systems. - -#### Installing Pilosa and Creating the Directory Structure - -If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](https://www.pilosa.com/docs/latest/installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](https://www.pilosa.com/docs/latest/installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](https://www.pilosa.com/docs/latest/installation/#build-from-source). - -After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line: -``` -pilosa --help -``` - -Let's create a directory for the tutorial to put all of our files and switch to that directory: -``` -mkdir $HOME/pilosa-tls-tutorial && cd $_ -``` - -#### Creating the TLS Certificate and Gossip Key - -Securing a Pilosa cluster consists of securing the communication between nodes using TLS and Gossip encryption. [Pilosa Enterprise](https://www.pilosa.com/enterprise/) additionally supports authentication and other security features, but those are not covered in this tutorial. - -The first step is acquiring an SSL certificate. You can buy a commercial certificate or retrieve a Let's Encrypt certificiate but we will be using a self signed certificate for practical reasons. Using self-signed certificates is not recommended in production, since it makes man in the middle attacks easy. - -The following command creates a 2048bit self-signed wildcard certificate for `*.pilosa.local` which expires 10 years later. - -``` -openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.crt -days 3650 -nodes -subj "/C=US/ST=Texas/L=Austin/O=Pilosa/OU=Com/CN=*.pilosa.local" -``` - -The command above creates two files in the current directory: -* `pilosa.local.crt` is the SSL certificate. -* `pilosa.local.key` is the private key file which must be kept as secret. - -Having created the SSL certificate, we can now create the gossip encryption key. Gossip encryption key file must be exactly 16, 24, or 32 bytes to select one of AES-128, AES-192, or AES-256 encryption. Reading random bytes from cryptographically secure `/dev/random` serves our purpose very well: -``` -head -c 32 /dev/random > pilosa.local.gossip32 -``` - -We now should have `pilosa.local.gossip32` in the current directory with 32 random bytes. - -#### Creating the Configuration Files - -Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes. - -Create `node1.config.toml` in the project directory and paste the following in it: - -```toml -# node1.config.toml - -data-dir = "node1_data" -bind = "https://01.pilosa.local:10501" - -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - -[tls] -certificate = "pilosa.local.crt" -key = "pilosa.local.key" -skip-verify = true - -[gossip] -seed = "01.pilosa.local:15000" -port = 15000 -key = "pilosa.local.gossip32" -``` - -Create `node2.config.toml` in the project directory and paste the following in it: - -```toml -# node2.config.toml - -data-dir = "node2_data" -bind = "https://02.pilosa.local:10502" - -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - -[tls] -certificate = "pilosa.local.crt" -key = "pilosa.local.key" -skip-verify = true - -[gossip] -seed = "01.pilosa.local:15000" -port = 16000 -key = "pilosa.local.gossip32" -``` - -Create `node3.config.toml` in the project directory and paste the following in it: - -```toml -# node3.config.toml - -data-dir = "node3_data" -bind = "https://03.pilosa.local:10503" - -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - -[tls] -certificate = "pilosa.local.crt" -key = "pilosa.local.key" -skip-verify = true - -[gossip] -seed = "01.pilosa.local:15000" -port = 17000 -key = "pilosa.local.gossip32" -``` - -Here is some explanation of the configuration items: -* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it. -* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes. -* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings. -* `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers. -* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. - -#### Final Touches Before Running the Cluster - -Before running the cluster, let's make sure that `01.pilosa.local`, `02.pilosa.local` and `03.pilosa.local` resolve to an IP address. If you are running the cluster on your computer, it is adequate to add them to your `/etc/hosts`. Below is one of the many ways of doing that (mind the `>>`): -``` -sudo sh -c 'printf "\n127.0.0.1 01.pilosa.local 02.pilosa.local 03.pilosa.local\n" >> /etc/hosts' -``` - -Ensure we can access the hosts in the cluster: -``` -ping -c 1 01.pilosa.local -ping -c 1 02.pilosa.local -ping -c 1 03.pilosa.local -``` - -If any of the commands above return `ping: unknown host`, make sure your `/etc/hosts` contains the failed hostname. - -#### Running the Cluster - -Let's open three terminal windows and run each node in its window. This will enable us to better observe what's happening on which node. - -Switch to the first terminal window, change to the project directory and start the first node: -``` -cd $HOME/pilosa-tls-tutorial -pilosa server -c node1.config.toml -``` - -Switch to the second terminal window, change to the project directory and start the second node: -``` -cd $HOME/pilosa-tls-tutorial -pilosa server -c node2.config.toml -``` - -Switch to the third terminal window, change to the project directory and start the third node: -``` -cd $HOME/pilosa-tls-tutorial -pilosa server -c node3.config.toml -``` - -Let's ensure that all three Pilosa servers are runnning and they are connected: -``` -curl -k --ipv4 https://01.pilosa.local:10501/status -``` - -The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL. - -All nodes should be in the `UP` state: -``` -{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}} -``` - -#### Running Queries - -Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame: -``` -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d '' -``` - -This will create index `sample-index` with default options. Let's create the frame now: -``` -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d '' -``` - -We just created frame `sample-frame` with default options. - -Let's run a `SetBit` query: -``` -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)' -``` - -Confirm that the bit was indeed set: -``` -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' -``` - -The same response should be returned when querying other nodes in the cluster: -``` -curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' -``` - -#### What's Next? - -Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa. diff --git a/docs/tutorials.md b/docs/tutorials.md index d304dc80d..4c2cdd7bd 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -2,346 +2,213 @@ title = "Tutorials" weight = 4 nav = [ - "Transportation", - "Chemical similarity search", + "How To Setup a Secure Cluster", ] +++ ## Tutorials -### Transportation +### How To Setup a Secure Cluster #### Introduction -New York City released an extremely detailed data set of over 1 billion taxi rides taken in the city - this data has become a popular target for analysis by tech bloggers and has been very well studied. For this reason, we thought it would be interesting to import this data to Pilosa in order to compare with other data stores and techniques on the exact same data set. +Pilosa supports encrypting the communication between and to nodes in a cluster using TLS. In this tutorial, we will be setting up a three node Pilosa cluster running on the same computer. The same steps can be used for a multi-computer cluster but that requires setting up firewalls and other platform-specific configuration which is out of the scope of this tutorial. -Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions). +This tutorial assumes that you are using a UNIX-like system, such as Linux or MacOS. [Windows Subsystem for Linux (WSL)](https://msdn.microsoft.com/en-us/commandline/wsl/about) works equally well on Windows 10 systems. -We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step. +#### Installing Pilosa and Creating the Directory Structure -After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa. +If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](https://www.pilosa.com/docs/latest/installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](https://www.pilosa.com/docs/latest/installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](https://www.pilosa.com/docs/latest/installation/#build-from-source). -#### Data Model - -The NYC taxi data is comprised of a number of csv files listed here: http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml. These data files have around 20 columns, about half of which are relevant to the benchmark queries we're looking at: - -* Distance: miles, floating point -* Fare: dollars, floating point -* Number of passengers: integer -* Dropoff location: latitude and longitude, floating point -* Pickup location: latitude and longitude, floating point -* Dropoff time: timestamp -* Pickup time: timestamp - -We import these fields, creating one or more Pilosa frames from each of them: - -frame |mapping -------------|--------------------- -cab_type |direct map of enum int → row ID -dist_miles |round(dist) → row ID -total_amount_dollars |round(dist) → row ID -passenger_count |direct map of integer value → row ID -drop_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID -drop_year |year(timestamp) → row ID -drop_month |month(timestamp) → row ID -drop_day |day(timestamp) → row ID -drop_time |time of day mapped to one of 48 half-hour buckets -pickup_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID -pickup_year |year(timestamp) → row ID -pickup_month |month(timestamp) → row ID -pickup_day |day(timestamp) → row ID -pickup_time |time of day mapped to one of 48 half-hour buckets → row ID - -We also created two extra frames that represent the duration and average speed of each ride: - -frame |mapping ---------------------|------------- -duration_minutes |round(drop_timestamp - pickup_timestamp) → row ID -speed_mph |round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID - -#### Mapping - -Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities. - -##### 0 columns → 1 frame - -cab_type: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant. - -##### 1 column → 1 frame - -The following three frames are mapped in a simple direct way from single columns of the original data. - -dist_miles: each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times. - -In PDK parlance, we define a Mapper, which is simply a function that returns integer row IDs. PDK has a number of predefined mappers that can be described with a few parameters. One of these is LinearFloatMapper, which applies a linear function to the input, and casts it to an integer, so the rounding is handled implicitly. In code: -```go -lfm := pdk.LinearFloatMapper{ - Min: -0.5, - Max: 3600.5, - Res: 3601, -} +After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line: +``` +pilosa --help ``` -`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three. - -This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`). -```go -pdk.BitMapper{ - Frame: "dist_miles", - Mapper: lfm, - Parsers: []pdk.Parser{pdk.FloatParser{}}, - Fields: []int{fields["trip_distance"]}, -}, +Let's create a directory for the tutorial to put all of our files and switch to that directory: +``` +mkdir $HOME/pilosa-tls-tutorial && cd $_ ``` -These same objects are represented in the JSON definition file: -```go -{ - "Fields": { - "Trip_distance": 10 - }, - "Mappers": [ - { - "Name": "lfm0", - "Min": -0.5, - "Max": 3600.5, - "Res": 3600 - } - ], - "BitMappers": [ - { - "Frame": "dist_miles", - "Mapper": { - "Name": "lfm0" - }, - "Parsers": [ - {"Name": "FloatParser"} - ], - "Fields": "Trip_distance" - } - ] -} -``` +#### Creating the TLS Certificate and Gossip Key -Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of BitMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names to column indices. We use these names in the BitMapper definitions to keep things human-readable. +Securing a Pilosa cluster consists of securing the communication between nodes using TLS and Gossip encryption. [Pilosa Enterprise](https://www.pilosa.com/enterprise/) additionally supports authentication and other security features, but those are not covered in this tutorial. -**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The BitMapper definition is very similar to the previous one. +The first step is acquiring an SSL certificate. You can buy a commercial certificate or retrieve a Let's Encrypt certificiate but we will be using a self signed certificate for practical reasons. Using self-signed certificates is not recommended in production, since it makes man in the middle attacks easy. -**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID. - -##### 1 column → multiple frames - -When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis. - -We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day". - -We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day". - -We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time. - -##### Multiple columns → 1 frame - -The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID. - -We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id. - -Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient. - -##### Complex mappings - -We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work: -```go -durm := pdk.CustomMapper{ - Func: func(fields ...interface{}) interface{} { - start := fields[0].(time.Time) - end := fields[1].(time.Time) - return end.Sub(start).Minutes() - }, - Mapper: lfm, -} -``` - -#### Import process - -After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process. - -#### Queries - -Now we can run some example queries. - -Count per cab type can be retrieved, sorted, with a single PQL call. +The following command creates a 2048bit self-signed wildcard certificate for `*.pilosa.local` which expires 10 years later. ``` -TopN(frame=cab_type) +openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.crt -days 3650 -nodes -subj "/C=US/ST=Texas/L=Austin/O=Pilosa/OU=Com/CN=*.pilosa.local" ``` -High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs. +The command above creates two files in the current directory: +* `pilosa.local.crt` is the SSL certificate. +* `pilosa.local.key` is the private key file which must be kept as secret. +Having created the SSL certificate, we can now create the gossip encryption key. Gossip encryption key file must be exactly 16, 24, or 32 bytes to select one of AES-128, AES-192, or AES-256 encryption. Reading random bytes from cryptographically secure `/dev/random` serves our purpose very well: ``` -TopN(frame=pickup_grid_id) +head -c 32 /dev/random > pilosa.local.gossip32 ``` -Average of total_amount per passenger_count can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average. +We now should have `pilosa.local.gossip32` in the current directory with 32 random bytes. -```python -queries = '' -pcounts = range(10) -for i in pcounts: - queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i -resp = requests.post(qurl, data=queries) +#### Creating the Configuration Files -average_amounts = [] -for pcount, topn in zip(pcounts, resp.json()['results']): - wsum = sum([r['count'] * r['key'] for r in topn]) - count = sum([r['count'] for r in topn]) - average_amounts.append(float(wsum)/count) +Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes. + +Create `node1.config.toml` in the project directory and paste the following in it: + +```toml +# node1.config.toml + +data-dir = "node1_data" +bind = "https://01.pilosa.local:10501" + +[cluster] +hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] + +[tls] +certificate = "pilosa.local.crt" +key = "pilosa.local.key" +skip-verify = true + +[gossip] +seed = "01.pilosa.local:15000" +port = 15000 +key = "pilosa.local.gossip32" ``` -For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb). +Create `node2.config.toml` in the project directory and paste the following in it: -### Chemical similarity search +```toml +# node2.config.toml -#### Overview +data-dir = "node2_data" +bind = "https://02.pilosa.local:10502" -The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals. +[cluster] +hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] -We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set. +[tls] +certificate = "pilosa.local.crt" +key = "pilosa.local.key" +skip-verify = true -Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient. -``` -T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B)) +[gossip] +seed = "01.pilosa.local:15000" +port = 16000 +key = "pilosa.local.gossip32" ``` -A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical. +Create `node3.config.toml` in the project directory and paste the following in it: -All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase +```toml +# node3.config.toml -#### Data model +data-dir = "node3_data" +bind = "https://03.pilosa.local:10503" -We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. +[cluster] +hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] -Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions. +[tls] +certificate = "pilosa.local.crt" +key = "pilosa.local.key" +skip-verify = true -Given a SMILES encoded molecule and a similarity threshold, we want to retrieve all molecule ids (or SMILES) that have a similarity percentage greater than or equal to the similarity threshold. For example, given a molecule with: -``` -SMILES = "IC=C1/CCC(C(=O)O1)c2cccc3ccccc23" -threshold = 90 +[gossip] +seed = "01.pilosa.local:15000" +port = 17000 +key = "pilosa.local.gossip32" ``` -return the set of molecules that have at least a 90% similarity with the given molecule. +Here is some explanation of the configuration items: +* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it. +* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes. +* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings. +* `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers. +* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. -The Inverse view swaps the rows and columns automatically to enable queries over either the chembl_id or fingerprint. +#### Final Touches Before Running the Cluster -Standard View is used to calculate similarity +Before running the cluster, let's make sure that `01.pilosa.local`, `02.pilosa.local` and `03.pilosa.local` resolve to an IP address. If you are running the cluster on your computer, it is adequate to add them to your `/etc/hosts`. Below is one of the many ways of doing that (mind the `>>`): ``` -Index: mole - View: Standard - Col: chembl_id - Frame: fingerprint - Row: position_id ("on" bit positions of a fingerprint) +sudo sh -c 'printf "\n127.0.0.1 01.pilosa.local 02.pilosa.local 03.pilosa.local\n" >> /etc/hosts' ``` -Inverse View is used for finding chembl_id based on given SMILES. -From a given SMILES, we use RDKit to convert it to fingerprints with "on" bit position. From "on" bit positions, we can search a list of chembl_ids that match the bit positions. To choose the right chembl_id, we need another query to Standard View then choose the right chembl_id which has the length that matches the given fingerprint's length after using RDKit to convert SMILES to fingerprint. +Ensure we can access the hosts in the cluster: ``` -Index: mole - View: Inverse - Col: position_id ("on" bit positions of a fingerprint) - Frame: fingerprint - Row: chembl_id +ping -c 1 01.pilosa.local +ping -c 1 02.pilosa.local +ping -c 1 03.pilosa.local ``` -After retrieving chembl_id from the Inverse View, we can use the Tanimoto coefficient to compare chembl_id with the entire data set of molecules. The result of this comparison is the list of `chembl_id`s that have a Tanimoto coefficient greater than the given threshold. +If any of the commands above return `ping: unknown host`, make sure your `/etc/hosts` contains the failed hostname. -#### Import process +#### Running the Cluster -To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key. +Let's open three terminal windows and run each node in its window. This will enable us to better observe what's happening on which node. -For the `mole` index, each row in the csv file has the format 'chembl_id, position_id' by running the following command from Chem-usecase: +Switch to the first terminal window, change to the project directory and start the first node: ``` -python import_from_sdf.py -p -file id_fingerprint.csv +cd $HOME/pilosa-tls-tutorial +pilosa server -c node1.config.toml ``` - -First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. -The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393 +Switch to the second terminal window, change to the project directory and start the second node: ``` -curl localhost:10101/index/mole \ - -X POST \ - -d '{"options": {"columnLabel": "position_id"}}' - -curl localhost:10101/index/mole/frame/fingerprint \ - -X POST \ - -d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}' - +cd $HOME/pilosa-tls-tutorial +pilosa server -c node2.config.toml ``` -Run the following commands to import the csv data into the `mole` index: +Switch to the third terminal window, change to the project directory and start the third node: ``` -pilosa import -d mole -f fingerprint id_fingerprint.csv +cd $HOME/pilosa-tls-tutorial +pilosa server -c node3.config.toml ``` -#### Queries - -Get chembl_id from a given SMILES: +Let's ensure that all three Pilosa servers are runnning and they are connected: ``` -python get_mol_fr_smile.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" +curl -k --ipv4 https://01.pilosa.local:10501/status ``` -Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get all chemlb_id that have positions are on, which following these steps: +The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL. -* Convert SMILES to fingerprint bit "on" positions - - ```python - from rdkit import Chem - from rdkit.Chem import AllChem - mol=Chem.MolFromSmiles("I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23") - fp = list(AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=4096).GetOnBits()) - ``` - -* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id - - ```python - bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] - bitmap_string = ', '.join(bit_maps) - intersection = "Intersect(%s)" % bitmap_string - mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"] - ``` - -* From list of chembl_id, query all "on" position from mol index, if the length of array of "on" position is matched to len(fp) then return that chembl_id, otherwise the given SMILES does not exist. - - ```python - for m in mole_ids: - mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"] - existed_mol = False - if len(mol) == len(fp): - found = m - existed_mol = True - break - ``` - -Retrieve molecule_ids that have similarity with SMILES="I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" and similarity threshold = 70% +All nodes should be in the `UP` state: ``` -python similar.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" -t 70 +{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}} ``` -Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN query to get all chemlb_id that have position is on, which following these steps: +#### Running Queries -* Get chembl_id from a SMILES (steps discussed above) - -* Query Pilosa’s TopN to get list of similarity chembl_id - ```python - query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)' - topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string) - ``` - -#### Benchmark - -To run benchmark for specific chembl_id for different similarity threshold at percentage of [50, 70, 75, 80, 85, 90], run following command: +Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame: ``` -python benchmarks.py -id 6223 +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d '' ``` -As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa. +This will create index `sample-index` with default options. Let's create the frame now: +``` +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d '' +``` -Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster +We just created frame `sample-frame` with default options. + +Let's run a `SetBit` query: +``` +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)' +``` + +Confirm that the bit was indeed set: +``` +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' +``` + +The same response should be returned when querying other nodes in the cluster: +``` +curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' +``` + +#### What's Next? + +Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa. From 0254a9ecbadf626979f68316c92fc65de9d478f5 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 14 Nov 2017 11:28:12 -0600 Subject: [PATCH 23/37] Replace html strong tags with markdown strong indicators --- docs/administration.md | 87 +++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 53 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index bc40fa41c..795d6f1c4 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -128,20 +128,20 @@ Note: This will only work when the replication factor is >= 2 Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions. -- Version: Version string of the build. -- Host: Host URI. -- Cluster: List of nodes in the Cluster. -- NumNodes: Number of nodes in the Cluster. -- NumCPU: Number of Cores per Node -- BSIEnabled: Bit Slice Index Frames in use. -- TimeQuantumEnabled: Time Quantum Frames in use. -- InverseEnabled: Inverse Frames in use. -- NumIndexes: Number of Indexes in the Cluster. -- NumFrames: Number of Frames in the Cluster. -- NumSlices: Number of Slices in the Cluster. -- NumViews: Number of Views in the Cluster. -- OpenFiles: Open file handle count. -- GoRoutines: Go routine count. +- **Version:** Version string of the build. +- **Host:** Host URI. +- **Cluster:** List of nodes in the Cluster. +- **NumNodes:** Number of nodes in the Cluster. +- **NumCPU:** Number of Cores per Node +- **BSIEnabled:** Bit Slice Index Frames in use. +- **TimeQuantumEnabled:** Time Quantum Frames in use. +- **InverseEnabled:** Inverse Frames in use. +- **NumIndexes:** Number of Indexes in the Cluster. +- **NumFrames:** Number of Frames in the Cluster. +- **NumSlices:** Number of Slices in the Cluster. +- **NumViews:** Number of Views in the Cluster. +- **OpenFiles:** Open file handle count. +- **GoRoutines:** Go routine count. You can opt-out of the Pilosa diagnostics reporting by setting either the command line configuration option `--metric.diagnostics=false`, use the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option. @@ -166,42 +166,23 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following: ##### Events We currently track the following events -Index: The creation of a new Index. - -Frame: The creation of a new Frame. - -MaxSlice: The Creation of a new Slice. - -SetBit: Count of set bits. - -ClearBit: Count of cleared bits. - -ImportBit: During a bulk data import this represents the count of bits created. - -SetRowAttrs: Count of Attributes set per row. - -SetColumnAttrs: Count of Attributes set per collumn. - -Bitmap: Count of Bitmap queries. - -TopN: Count of TopN queries. - -Union: Count of Union queries. - -Intersection: Count of Intersection queries. - -Difference: Count of Difference queries. - -Count: Count of Count queries. - -Range: Count of Range queries. - -Snapshot: Event count when the snapshot process is triggered. - -BlockRepair: Count of data blocks that were out of sync and repaired. - -Garbage Collection: Event count when Garbage Collection occurs. - -Goroutines: Number of running Goroutines. - -OpenFiles: Number of open file handles associated with running Pilosa process ID. +- **Index:** The creation of a new Index. +- **Frame:** The creation of a new Frame. +- **MaxSlice:** The Creation of a new Slice. +- **SetBit:** Count of set bits. +- **ClearBit:** Count of cleared bits. +- **ImportBit:** During a bulk data import this represents the count of bits created. +- **SetRowAttrs:** Count of Attributes set per row. +- **SetColumnAttrs:** Count of Attributes set per collumn. +- **Bitmap:** Count of Bitmap queries. +- **TopN:** Count of TopN queries. +- **Union:** Count of Union queries. +- **Intersection:** Count of Intersection queries. +- **Difference:** Count of Difference queries. +- **Count:** Count of Count queries. +- **Range:** Count of Range queries. +- **Snapshot:** Event count when the snapshot process is triggered. +- **BlockRepair:** Count of data blocks that were out of sync and repaired. +- **Garbage Collection:** Event count when Garbage Collection occurs. +- **Goroutines:** Number of running Goroutines. +- **OpenFiles:** Number of open file handles associated with running Pilosa process ID. From 8fdaea5777b99734d0903fb82a8d5b8d5196a1c7 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 14 Nov 2017 20:41:00 +0300 Subject: [PATCH 24/37] fixed tests --- handler_test.go | 12 ++++++------ index_test.go | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/handler_test.go b/handler_test.go index 280467cfc..796782402 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1587,7 +1587,7 @@ func TestHandler_CreateInput(t *testing.T) { } inputBody := []byte(` [{ - "columnID": 1, + "id": 1, "cabType": "yellow", "distanceMiles": 8, "withPet": true, @@ -1668,14 +1668,14 @@ func TestInput_JSON(t *testing.T) { err string }{ {json: `[{ - "columnID": 1, + "id": 1, "cabType": "yellow", "distanceMiles": 8, "nofield": true }]`, err: "field not found: nofield"}, {json: `[{ - "columnID": "abc", + "id": "abc", "cabType": "yellow", "distanceMiles": 8, "withPet": true @@ -1688,21 +1688,21 @@ func TestInput_JSON(t *testing.T) { }]`, err: "primary key does not exist"}, {json: `[{ - "columnID": 1, + "id": 1, "cabType": "yellow", "distanceMiles": 8, "withPet": true }`, err: "unexpected EOF"}, {json: `[{ - "columnID": 1, + "id": 1, "cabType": "yellow", "distanceMiles": 8, "noFrame": 1 }]`, err: "Frame not found: foo"}, {json: `[{ - "columnID": 1, + "id": 1, "cabType": "yellow", "distanceMiles": 8, "time_value": 12345 diff --git a/index_test.go b/index_test.go index dbfc939b5..5e44a53a4 100644 --- a/index_test.go +++ b/index_test.go @@ -309,14 +309,14 @@ func TestIndex_CreateInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } else if inputDef.Frames()[0].Name != frames.Name { t.Fatalf("unexpected input definition frames %v", inputDef.Frames()) - } else if inputDef.Fields()[0].Name != pilosa.DefaultColumnLabel { + } else if inputDef.Fields()[0].Name != field.Name { t.Fatalf("unexpected input definition actions %v", inputDef.Fields()) } } From e281d9038fea18c1c334b52815368ac9a8e702bc Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 14 Nov 2017 13:28:00 -0600 Subject: [PATCH 25/37] Temporarily disable Go master CI as builds are failing due to possible Go bug (See #956) --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fd97d53e8..2377d30f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: go go: - 1.8 - 1.9 - - master env: global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" From a17b1b5b277ad6b516366d26037210b224cb9931 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 14 Nov 2017 14:57:32 -0600 Subject: [PATCH 26/37] Simplify TestCountOpenFiles: Fragile in some environments due to unpredictable open file count --- server/server_test.go | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/server/server_test.go b/server/server_test.go index 849f12b4b..19f581e19 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -25,7 +25,6 @@ import ( "net" "net/http" "os" - "path/filepath" "reflect" "runtime" "sort" @@ -373,26 +372,10 @@ func tempMkdir(t *testing.T) string { // Ensure the file handle count is working func TestCountOpenFiles(t *testing.T) { // Windows is not supported yet - supported := []string{"darwin", "linux", "unix", "freebsd"} - sort.Strings(supported) - i := sort.Search(len(supported), - func(i int) bool { return supported[i] >= runtime.GOOS }) - if i == len(supported) { + if runtime.GOOS == "windows" { return } - - // Create directory store temp file - testDir := tempMkdir(t) - defer os.RemoveAll(testDir) - - count := pilosa.CountOpenFiles() - testFile := filepath.Join(testDir, "test.txt") - _, err := os.Create(testFile) - if err != nil { - t.Fatalf("create test file failed: %s", err) - } - - if pilosa.CountOpenFiles() < count+1 { + if pilosa.CountOpenFiles() == 0 { t.Error("Invalid open file handle count") } } From 956472fabd1de2155f65c6e8029f77104a2d99d1 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 15 Nov 2017 08:28:23 -0600 Subject: [PATCH 27/37] Release v0.8.0 --- CHANGELOG.md | 48 ++++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 2 +- docs/installation.md | 12 +++++------ 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb7141d75..0a54a5b73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,51 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [0.8.0] - 2017-11-15 + +This version contains 31 contributions from 8 contributors. There are 84 files changed, 3,732 insertions, and 1,428 deletions. + +### Added + +- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895)) +- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933)) +- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938)) +- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942)) +- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867)) +- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898)) +- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889)) +- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881)) +- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955)) + +### Changed + +- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943)) +- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884)) +- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892)) +- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945)) +- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) + +### Fixed + +- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941)) +- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934)) +- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919)) +- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887)) +- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882)) +- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949)) + +### Performance + +- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874)) + +## [0.7.2] - 2017-11-15 + +This version contains 1 contribution from 1 contributor. There is 1 file changed, 16 insertions, and 1 deletion. + +### Changed + +- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920)) + ## [0.7.1] - 2017-10-09 This version contains 3 contributions from 3 contributors. There are 14 files changed, 221 insertions, and 52 deletions. @@ -163,3 +208,6 @@ This version contains 53 contributions from 13 contributors (including 4 volunte [Unreleased]: https://github.com/pilosa/pilosa/compare/v0.5...HEAD [0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4 [0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5 +[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6 +[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7 +[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8 diff --git a/Dockerfile b/Dockerfile index f129ce32c..c5f6d1498 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.9.1 as builder +FROM golang:1.9.2 as builder ARG ldflags='' diff --git a/docs/installation.md b/docs/installation.md index c0455ac32..33528f969 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -74,19 +74,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.7.1/pilosa-v0.7.1-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.0/pilosa-v0.8.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.7.1-darwin-amd64.tar.gz + tar xfz pilosa-v0.8.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.7.1-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.8.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -228,19 +228,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.7.1/pilosa-v0.7.1-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.0/pilosa-v0.8.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.7.1-linux-amd64.tar.gz + tar xfz pilosa-v0.8.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.7.1-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.8.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: From a9a1ec251cc372149da3ff49ee66082e3c5d8c29 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 15 Nov 2017 10:53:35 -0600 Subject: [PATCH 28/37] Re-enable CI on Go master, but allow failures. See #956. --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 2377d30f1..c368fbd49 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: go go: - 1.8 - 1.9 + - master env: global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" @@ -19,6 +20,9 @@ deploy: skip_cleanup: true on: branch: master +matrix: + allow_failures: + - go: master notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" From 09a4a72720de4316151cfc7d55d03aeedde92e26 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 15 Nov 2017 13:07:51 -0600 Subject: [PATCH 29/37] fixed version check when local is greater than pilosa.com --- diagnostics/diagnostics.go | 4 ++-- diagnostics/diagnostics_test.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index 6bcd433d3..b1453db22 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -172,9 +172,9 @@ func (d *Diagnostics) CompareVersion(value string) error { if localVersion[0] < currentVersion[0] { //Major return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value) - } else if localVersion[1] < currentVersion[1] { // Minor + } else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value) - } else if localVersion[2] < currentVersion[2] { // Patch + } else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value) } diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 6ad0cb773..8e3a7cce4 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -96,6 +96,11 @@ func TestDiagnosticsVersion_Compare(t *testing.T) { if err != nil { t.Fatalf("Versions should match") } + d.SetVersion("v1.7.0") + err = d.CompareVersion("0.7.2") + if err != nil { + t.Fatalf("Local version is greater") + } } func TestDiagnosticsVersion_Check(t *testing.T) { From 5865a2cd71e6677e7235682c3b4d050065ee2887 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 15 Nov 2017 13:19:13 -0600 Subject: [PATCH 30/37] Fix CountOpenFiles() fatal crash --- server.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/server.go b/server.go index 0d55e7c47..a21b408c3 100644 --- a/server.go +++ b/server.go @@ -534,7 +534,10 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes())) s.diagnostics.Set("NumFrames", numFrames) s.diagnostics.Set("NumSlices", numSlices) - s.diagnostics.Set("OpenFiles", CountOpenFiles()) + openFiles, err := CountOpenFiles() + if err == nil { + s.diagnostics.Set("OpenFiles", openFiles) + } s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) s.diagnostics.CheckVersion() s.diagnostics.Flush() @@ -584,8 +587,11 @@ func (s *Server) monitorRuntime() { // Record the number of go routines. s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) + openFiles, err := CountOpenFiles() // Open File handles. - s.Holder.Stats.Gauge("OpenFiles", float64(CountOpenFiles()), 1.0) + if err == nil { + s.Holder.Stats.Gauge("OpenFiles", float64(openFiles), 1.0) + } // Runtime memory metrics. runtime.ReadMemStats(&m) @@ -606,26 +612,24 @@ func (s *Server) createDefaultClient() { } // CountOpenFiles on operating systems that support lsof. -func CountOpenFiles() int { - count := 0 - +func CountOpenFiles() (int, error) { switch runtime.GOOS { case "darwin", "linux", "unix", "freebsd": // -b option avoid kernel blocks pid := os.Getpid() out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -b -p %v", pid)).Output() if err != nil { - log.Fatal(err) + return 0, fmt.Errorf("calling lsof: %s", err) } // only count lines with our pid, avoiding warning messages from -b lines := strings.Split(string(out), strconv.Itoa(pid)) - count = len(lines) + return len(lines), nil case "windows": // TODO: count open file handles on windows + return 0, errors.New("CountOpenFiles() on Windows is not supported") default: - + return 0, errors.New("CountOpenFiles() on this OS is not supported") } - return count } // StatusHandler specifies two methods which an object must implement to share From dc7dd2695591dddb0564557f217a1ccc2b84b03c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 15 Nov 2017 13:38:55 -0600 Subject: [PATCH 31/37] Fix test for CountOpenFiles --- server/server_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/server/server_test.go b/server/server_test.go index 19f581e19..d22abbd1b 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -373,10 +373,14 @@ func tempMkdir(t *testing.T) string { func TestCountOpenFiles(t *testing.T) { // Windows is not supported yet if runtime.GOOS == "windows" { - return + t.Skip("Skipping unsupported CountOpenFiles test on Windows.") } - if pilosa.CountOpenFiles() == 0 { - t.Error("Invalid open file handle count") + count, err := pilosa.CountOpenFiles() + if err != nil { + t.Errorf("CountOpenFiles failed: %s", err) + } + if count == 0 { + t.Error("CountOpenFiles returned invalid value 0.") } } From c0a323bfae54d6a11f733be8f963786723a0b2a3 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 17 Nov 2017 14:37:33 +0300 Subject: [PATCH 32/37] updated Go client sample --- docs/client-libraries.md | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/docs/client-libraries.md b/docs/client-libraries.md index cad26a18f..67e36b5dc 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -46,18 +46,18 @@ func main() { } // We need to refer to indexes and frames before we can use them in a query. - repository, _ := schema.Index("repository", nil) - stargazer, _ := repository.Frame("stargazer", nil) - language, _ := repository.Frame("language", nil) + repository, _ := schema.Index("repository") + stargazer, _ := repository.Frame("stargazer") + language, _ := repository.Frame("language") var response *pilosa.QueryResponse // Which repositories did user 14 star: - response, _ = client.Query(stargazer.Bitmap(14), nil) + response, _ = client.Query(stargazer.Bitmap(14)) fmt.Println("User 14 starred: ", response.Result().Bitmap.Bits) // What are the top 5 languages in the sample data? - response, err = client.Query(language.TopN(5), nil) + response, err = client.Query(language.TopN(5)) languageIDs := []uint64{} for _, item := range response.Result().CountItems { languageIDs = append(languageIDs, item.ID) @@ -68,16 +68,14 @@ func main() { response, _ = client.Query( repository.Intersect( stargazer.Bitmap(14), - stargazer.Bitmap(19)), - nil) + stargazer.Bitmap(19))) fmt.Println("Both user 14 and 19 starred:", response.Result().Bitmap.Bits) // Which repositories were starred by user 14 or 19: response, _ = client.Query( repository.Union( stargazer.Bitmap(14), - stargazer.Bitmap(19)), - nil) + stargazer.Bitmap(19))) fmt.Println("User 14 or 19 starred:", response.Result().Bitmap.Bits) // Which repositories were starred by user 14 or 19 and were written in language 1: @@ -87,12 +85,11 @@ func main() { stargazer.Bitmap(14), stargazer.Bitmap(19), ), - language.Bitmap(1), - ), nil) + language.Bitmap(1))) fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Bitmap.Bits) // Set user 99999 as a stargazer for repository 77777? - client.Query(stargazer.SetBit(99999, 77777), nil) + client.Query(stargazer.SetBit(99999, 77777)) } ``` From dff312b92798dd49cdba97d62149c63a6e69f2c1 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Sun, 19 Nov 2017 17:22:29 -0600 Subject: [PATCH 33/37] language.txt -> languages.txt --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 84c5fd0bf..f950f1d0d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -100,7 +100,7 @@ docker cp language.csv pilosa:/language.csv docker exec -it pilosa /pilosa import -i repository -f language /language.csv ``` -Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `language.txt` to see the mapping for languages. +Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `languages.txt` to see the mapping for languages. ### Input Definition Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data. From c569cbe071acee4b518f2e8e0b0863925693aaa7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 21 Nov 2017 14:30:14 -0600 Subject: [PATCH 34/37] Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c5f6d1498..26a309d9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,4 @@ EXPOSE 10101 VOLUME /data ENTRYPOINT ["/pilosa"] -CMD ["server", "--data-dir", "/data"] +CMD ["server", "--data-dir", "/data", "--bind", "0.0.0.0:10101"] From 1b5671972af38d6e813c5b686a4ff73740781b0a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 21 Nov 2017 20:08:45 -0600 Subject: [PATCH 35/37] Add protocol to Dockerfile --bind argument. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 26a309d9c..b145465b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,4 @@ EXPOSE 10101 VOLUME /data ENTRYPOINT ["/pilosa"] -CMD ["server", "--data-dir", "/data", "--bind", "0.0.0.0:10101"] +CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"] From f051f7ccea6674b2ebe168f72b6e5923eebb1945 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 1 Dec 2017 16:00:12 -0600 Subject: [PATCH 36/37] refactored httpclient handling --- client.go | 30 ++++-------------------------- client_test.go | 34 +++++++++++++++++++++------------- ctl/common.go | 8 ++++---- executor.go | 24 +++++++++++------------- fragment.go | 11 ++++++----- handler.go | 8 ++++---- holder.go | 21 +++++++++++---------- holder_test.go | 11 ++++++----- server.go | 34 +++++++++++++++++++++++++--------- server/server.go | 13 +++++++++++-- server/server_test.go | 6 +++--- test/client.go | 6 ++++-- test/executor.go | 2 +- 13 files changed, 111 insertions(+), 97 deletions(-) diff --git a/client.go b/client.go index 666447586..e44d9c5de 100644 --- a/client.go +++ b/client.go @@ -25,7 +25,6 @@ import ( "io/ioutil" "log" "math/rand" - "net" "net/http" "net/url" "sort" @@ -46,14 +45,13 @@ type ClientOptions struct { // InternalHTTPClient represents a client to the Pilosa cluster. type InternalHTTPClient struct { defaultURI *URI - options *ClientOptions // The client to use for HTTP communication. HTTPClient *http.Client } // NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host. -func NewInternalHTTPClient(host string, options *ClientOptions) (*InternalHTTPClient, error) { +func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTTPClient, error) { if host == "" { return nil, ErrHostRequired } @@ -63,34 +61,14 @@ func NewInternalHTTPClient(host string, options *ClientOptions) (*InternalHTTPCl return nil, err } - client := NewInternalHTTPClientFromURI(uri, options) + client := NewInternalHTTPClientFromURI(uri, remoteClient) return client, nil } -func NewInternalHTTPClientFromURI(defaultURI *URI, options *ClientOptions) *InternalHTTPClient { - if options == nil { - options = &ClientOptions{} - } - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 1000, - MaxIdleConnsPerHost: 200, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - if options.TLS != nil { - transport.TLSClientConfig = options.TLS - } - client := &http.Client{Transport: transport} +func NewInternalHTTPClientFromURI(defaultURI *URI, remoteClient *http.Client) *InternalHTTPClient { return &InternalHTTPClient{ defaultURI: defaultURI, - HTTPClient: client, + HTTPClient: remoteClient, } } diff --git a/client_test.go b/client_test.go index 613d37659..8feec09ea 100644 --- a/client_test.go +++ b/client_test.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "net/http" "reflect" "testing" @@ -43,6 +44,13 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { return server, hldr } +var defaultClient *http.Client + +func init() { + defaultClient = pilosa.GetHTTPClient(nil) + +} + // Test distributed TopN Row count across 3 nodes. func TestClient_MultiNode(t *testing.T) { cluster := test.NewCluster(3) @@ -54,7 +62,7 @@ func TestClient_MultiNode(t *testing.T) { } s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[0].Holder e.Scheme = cluster.Nodes[0].Scheme e.Host = cluster.Nodes[0].Host @@ -62,7 +70,7 @@ func TestClient_MultiNode(t *testing.T) { return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[1].Holder e.Scheme = cluster.Nodes[1].Scheme e.Host = cluster.Nodes[1].Host @@ -70,7 +78,7 @@ func TestClient_MultiNode(t *testing.T) { return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[2].Holder e.Scheme = cluster.Nodes[2].Scheme e.Host = cluster.Nodes[2].Host @@ -135,9 +143,9 @@ func TestClient_MultiNode(t *testing.T) { // Connect to each node to compare results. client := make([]*test.Client, 3) - client[0] = test.MustNewClient(s[0].Host()) - client[1] = test.MustNewClient(s[1].Host()) - client[2] = test.MustNewClient(s[2].Host()) + client[0] = test.MustNewClient(s[0].Host(), defaultClient) + client[1] = test.MustNewClient(s[1].Host(), defaultClient) + client[2] = test.MustNewClient(s[2].Host(), defaultClient) topN := 4 queryRequest := &internal.QueryRequest{ @@ -218,7 +226,7 @@ func TestClient_Import(t *testing.T) { s.Handler.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, @@ -269,7 +277,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) { s.Handler.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, @@ -318,7 +326,7 @@ func TestClient_ImportValue(t *testing.T) { s.Handler.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{ {ColumnID: 1, Value: -10}, {ColumnID: 2, Value: 20}, @@ -355,7 +363,7 @@ func TestClient_BackupRestore(t *testing.T) { s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) // Backup from frame. var buf bytes.Buffer @@ -420,7 +428,7 @@ func TestClient_BackupInverseView(t *testing.T) { s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) // Backup from frame. var buf bytes.Buffer @@ -457,7 +465,7 @@ func TestClient_BackupInvalidView(t *testing.T) { s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) // Backup from frame. var buf bytes.Buffer @@ -487,7 +495,7 @@ func TestClient_FragmentBlocks(t *testing.T) { s.Handler.Holder = hldr.Holder // Retrieve blocks. - c := test.MustNewClient(s.Host()) + c := test.MustNewClient(s.Host(), defaultClient) blocks, err := c.FragmentBlocks(context.Background(), "i", "f", pilosa.ViewStandard, 0) if err != nil { t.Fatal(err) diff --git a/ctl/common.go b/ctl/common.go index dc64c0bee..826462b8c 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -2,6 +2,7 @@ package ctl import ( "crypto/tls" + "github.com/pilosa/pilosa" "github.com/spf13/pflag" ) @@ -22,19 +23,18 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP // CommandClient returns a pilosa.InternalHTTPClient for the command func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error) { tlsConfig := cmd.TLSConfiguration() - var clientOptions *pilosa.ClientOptions + var TLSConfig *tls.Config if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" { cert, err := tls.LoadX509KeyPair(tlsConfig.CertificatePath, tlsConfig.CertificateKeyPath) if err != nil { return nil, err } - TLSConfig := &tls.Config{ + TLSConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: tlsConfig.SkipVerify, } - clientOptions = &pilosa.ClientOptions{TLS: TLSConfig} } - client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), clientOptions) + client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), pilosa.GetHTTPClient(TLSConfig)) if err != nil { return nil, err } diff --git a/executor.go b/executor.go index 2f5acf3c0..a617eb184 100644 --- a/executor.go +++ b/executor.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "net/http" "sort" "time" @@ -51,12 +52,9 @@ type Executor struct { } // NewExecutor returns a new instance of Executor. -func NewExecutor(clientOptions *ClientOptions) *Executor { - if clientOptions == nil { - clientOptions = &ClientOptions{} - } +func NewExecutor(remoteClient *http.Client) *Executor { return &Executor{ - client: NewInternalHTTPClientFromURI(nil, clientOptions), + client: NewInternalHTTPClientFromURI(nil, remoteClient), } } @@ -968,7 +966,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql } // Forward call to remote node otherwise. - if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { + if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { return false, err } else { ret = res[0].(bool) @@ -1074,7 +1072,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C } // Forward call to remote node otherwise. - if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { + if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { return false, err } else { ret = res[0].(bool) @@ -1141,7 +1139,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) + _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) resp <- err }(node) } @@ -1199,7 +1197,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) + _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) resp <- err }(node) } @@ -1286,7 +1284,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, index, &pql.Query{Calls: calls}, nil, opt) + _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil, opt) resp <- err }(node) } @@ -1345,7 +1343,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) + _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) resp <- err }(node) } @@ -1361,7 +1359,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p } // exec executes a PQL query remotely for a set of slices on a node. -func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { +func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { // Encode request object. pbreq := &internal.QueryRequest{ Query: q.String(), @@ -1511,7 +1509,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod if n.Host == e.Host { resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) } else if !opt.Remote { - results, err := e.exec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) + results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) if len(results) > 0 { resp.result = results[0] } diff --git a/fragment.go b/fragment.go index e5fbb7988..073bdfa84 100644 --- a/fragment.go +++ b/fragment.go @@ -28,6 +28,7 @@ import ( "io" "io/ioutil" "log" + "net/http" "os" "sort" "sync" @@ -1677,9 +1678,9 @@ func (h *blockHasher) WriteValue(v uint64) { type FragmentSyncer struct { Fragment *Fragment - Host string - Cluster *Cluster - ClientOptions *ClientOptions + Host string + Cluster *Cluster + RemoteClient *http.Client Closing <-chan struct{} } @@ -1714,7 +1715,7 @@ func (s *FragmentSyncer) SyncFragment() error { } // Retrieve remote blocks. - client, err := NewInternalHTTPClient(node.Host, s.ClientOptions) + client, err := NewInternalHTTPClient(node.Host, s.RemoteClient) if err != nil { return err } @@ -1793,7 +1794,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { return nil } - client, err := NewInternalHTTPClient(node.Host, s.ClientOptions) + client, err := NewInternalHTTPClient(node.Host, s.RemoteClient) if err != nil { return err } diff --git a/handler.go b/handler.go index f3d9fe9d3..ad9e02e2e 100644 --- a/handler.go +++ b/handler.go @@ -56,9 +56,9 @@ type Handler struct { StatusHandler StatusHandler // Local hostname & cluster configuration. - URI *URI - Cluster *Cluster - ClientOptions *ClientOptions + URI *URI + Cluster *Cluster + RemoteClient *http.Client Router *mux.Router @@ -1506,7 +1506,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) } // Create a client for the remote cluster. - client := NewInternalHTTPClientFromURI(host, h.ClientOptions) + client := NewInternalHTTPClientFromURI(host, h.RemoteClient) // Determine the maximum number of slices. maxSlices, err := client.MaxSliceByIndex(r.Context()) diff --git a/holder.go b/holder.go index 4241dcd85..7cbb8ca42 100644 --- a/holder.go +++ b/holder.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "log" + "net/http" "os" "path/filepath" "sort" @@ -430,9 +431,9 @@ func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.Lstd type HolderSyncer struct { Holder *Holder - URI *URI - Cluster *Cluster - ClientOptions *ClientOptions + URI *URI + Cluster *Cluster + RemoteClient *http.Client // Signals that the sync should stop. Closing <-chan struct{} @@ -518,7 +519,7 @@ func (s *HolderSyncer) syncIndex(index string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) { - client, err := NewInternalHTTPClient(node.Host, s.ClientOptions) + client, err := NewInternalHTTPClient(node.Host, s.RemoteClient) if err != nil { return err } @@ -563,7 +564,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) { - client, err := NewInternalHTTPClient(node.Host, s.ClientOptions) + client, err := NewInternalHTTPClient(node.Host, s.RemoteClient) if err != nil { return err } @@ -616,11 +617,11 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err // Sync fragments together. fs := FragmentSyncer{ - Fragment: frag, - Host: s.URI.HostPort(), - Cluster: s.Cluster, - Closing: s.Closing, - ClientOptions: s.ClientOptions, + Fragment: frag, + Host: s.URI.HostPort(), + Cluster: s.Cluster, + Closing: s.Closing, + RemoteClient: s.RemoteClient, } if err := fs.SyncFragment(); err != nil { return err diff --git a/holder_test.go b/holder_test.go index 7020a1b8c..6cbfdf150 100644 --- a/holder_test.go +++ b/holder_test.go @@ -320,7 +320,7 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { cluster := test.NewCluster(2) - + client := pilosa.GetHTTPClient(nil) // Create a local holder. hldr0 := test.MustOpenHolder() defer hldr0.Close() @@ -332,7 +332,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { defer s.Close() s.Handler.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e := pilosa.NewExecutor(client) e.Holder = hldr1.Holder e.Scheme = cluster.Nodes[1].Scheme e.Host = cluster.Nodes[1].Host @@ -400,9 +400,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { t.Fatal(err) } syncer := pilosa.HolderSyncer{ - Holder: hldr0.Holder, - URI: uri, - Cluster: cluster, + Holder: hldr0.Holder, + URI: uri, + Cluster: cluster, + RemoteClient: pilosa.GetHTTPClient(nil), } if err := syncer.SyncHolder(); err != nil { diff --git a/server.go b/server.go index a21b408c3..ffaad0b71 100644 --- a/server.go +++ b/server.go @@ -58,6 +58,7 @@ type Server struct { Handler *Handler Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver + RemoteClient *http.Client // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -167,10 +168,10 @@ func (s *Server) Open() error { } // Create default HTTP client - s.createDefaultClient() + s.createDefaultClient(s.RemoteClient) // Create executor for executing queries. - e := NewExecutor(&ClientOptions{TLS: s.TLS}) + e := NewExecutor(s.RemoteClient) e.Holder = s.Holder e.Scheme = s.URI.Scheme() e.Host = s.URI.HostPort() @@ -229,6 +230,25 @@ func (s *Server) Addr() net.Addr { } return s.ln.Addr() } +func GetHTTPClient(t *tls.Config) *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 200, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + if t != nil { + transport.TLSClientConfig = t + } + return &http.Client{Transport: transport} +} // Logger returns a logger that writes to LogOutput func (s *Server) Logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } @@ -256,7 +276,7 @@ func (s *Server) monitorAntiEntropy() { syncer.URI = s.URI syncer.Cluster = s.Cluster syncer.Closing = s.closing - syncer.ClientOptions = &ClientOptions{TLS: s.TLS} + syncer.RemoteClient = s.RemoteClient // Sync holders. if err := syncer.SyncHolder(); err != nil { @@ -603,12 +623,8 @@ func (s *Server) monitorRuntime() { } } -func (s *Server) createDefaultClient() { - transport := &http.Transport{} - if s.TLS != nil { - transport.TLSClientConfig = s.TLS - } - s.defaultClient = NewInternalHTTPClientFromURI(nil, &ClientOptions{TLS: s.TLS}) +func (s *Server) createDefaultClient(remoteClient *http.Client) { + s.defaultClient = NewInternalHTTPClientFromURI(nil, remoteClient) } // CountOpenFiles on operating systems that support lsof. diff --git a/server/server.go b/server/server.go index 7cc8c5dec..a5f17f6d7 100644 --- a/server/server.go +++ b/server/server.go @@ -31,10 +31,11 @@ import ( "crypto/tls" + "io/ioutil" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" - "io/ioutil" ) func init() { @@ -161,6 +162,7 @@ func (m *Command) SetupServer() error { m.Server.MaxWritesPerRequest = m.Config.MaxWritesPerRequest // Setup TLS + var TLSConfig *tls.Config if uri.Scheme() == "https" { if m.Config.TLS.CertificatePath == "" { return errors.New("certificate path is required for TLS sockets") @@ -176,8 +178,15 @@ func (m *Command) SetupServer() error { Certificates: []tls.Certificate{cert}, InsecureSkipVerify: m.Config.TLS.SkipVerify, } - m.Server.Handler.ClientOptions = &pilosa.ClientOptions{TLS: m.Server.TLS} + + // TODO Review this location + + TLSConfig = m.Server.TLS + } + c := pilosa.GetHTTPClient(TLSConfig) + m.Server.RemoteClient = c + m.Server.Handler.RemoteClient = c // Set internal port (string). gossipPortStr := pilosa.DefaultGossipPort diff --git a/server/server_test.go b/server/server_test.go index d22abbd1b..28886bc67 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -50,7 +50,7 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), nil) + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } @@ -323,7 +323,7 @@ func TestMain_FrameRestore(t *testing.T) { defer m2.Close() // Import from first cluster. - client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), nil) + client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } else if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { @@ -672,7 +672,7 @@ func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } // Client returns a client to connect to the program. func (m *Main) Client() *pilosa.InternalHTTPClient { - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), nil) + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) if err != nil { panic(err) } diff --git a/test/client.go b/test/client.go index 4ea2fbbad..5eac8cb79 100644 --- a/test/client.go +++ b/test/client.go @@ -1,6 +1,8 @@ package test import ( + "net/http" + "github.com/pilosa/pilosa" ) @@ -10,8 +12,8 @@ type Client struct { } // MustNewClient returns a new instance of Client. Panic on error. -func MustNewClient(host string) *Client { - c, err := pilosa.NewInternalHTTPClient(host, nil) +func MustNewClient(host string, h *http.Client) *Client { + c, err := pilosa.NewInternalHTTPClient(host, h) if err != nil { panic(err) } diff --git a/test/executor.go b/test/executor.go index be370908d..1cddfd521 100644 --- a/test/executor.go +++ b/test/executor.go @@ -15,7 +15,7 @@ type Executor struct { // NewExecutor returns a new instance of Executor. // The executor always matches the hostname of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - executor := pilosa.NewExecutor(nil) + executor := pilosa.NewExecutor(pilosa.GetHTTPClient(nil)) e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster From e7d64c4f48e39f15ca55b0cd3b5c9fb3b5fca0f5 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 4 Dec 2017 12:17:41 -0600 Subject: [PATCH 37/37] limit httpclient instances on executor tests --- test/executor.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/executor.go b/test/executor.go index 1cddfd521..f00692361 100644 --- a/test/executor.go +++ b/test/executor.go @@ -1,6 +1,7 @@ package test import ( + "net/http" "strings" "github.com/pilosa/pilosa" @@ -12,10 +13,16 @@ type Executor struct { *pilosa.Executor } +var remoteClient *http.Client + +func init() { + remoteClient = pilosa.GetHTTPClient(nil) +} + // NewExecutor returns a new instance of Executor. // The executor always matches the hostname of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - executor := pilosa.NewExecutor(pilosa.GetHTTPClient(nil)) + executor := pilosa.NewExecutor(remoteClient) e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster