From fec0337630c07a0938e6f93211d7a41b416ec025 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 5 Feb 2021 12:16:08 -0600 Subject: [PATCH 01/29] add draft trial version of molecula --- Makefile | 11 ++++++++-- go.mod | 1 + go.sum | 2 ++ server/server.go | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ version.go | 3 ++- 5 files changed, 67 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 2e6cc7331..8a724c282 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice release-build-trial CLONE_URL=github.com/pilosa/pilosa MOD_VERSION=v2 @@ -11,7 +11,7 @@ BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) -LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)" +LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT) -X github.com/pilosa/pilosa/v2.TrialDeadline=$(TRIAL_DEADLINE)" GO_VERSION=1.14.10 RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) @@ -98,6 +98,13 @@ cover-viz: cover build: go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa +# Create a single release trial build under the build directory +release-build-trial: + $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-trial-$(VERSION_ID)/pilosa" RELEASE=1 + cp NOTICE README.md build/pilosa-trial-$(VERSION_ID) + tar -cvz -C build -f build/pilosa-trial-$(VERSION_ID).tar.gz pilosa-trial-$(VERSION_ID)/ + @echo Created release trial build: build/pilosa-trial-$(VERSION_ID).tar.gz + # Create a single release build under the build directory release-build: $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1 diff --git a/go.mod b/go.mod index f0d161c95..dcd6b603a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 + github.com/beevik/ntp v0.3.0 github.com/benbjohnson/immutable v0.3.0 github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect diff --git a/go.sum b/go.sum index b9d5e44d7..f1ad4ecc5 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= +github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/immutable v0.3.0 h1:TVRhuZx2wG9SZ0LRdqlbs9S5BZ6Y24hJEHTCgWHZEIw= github.com/benbjohnson/immutable v0.3.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= diff --git a/server/server.go b/server/server.go index 530e4102a..1e9a8d053 100644 --- a/server/server.go +++ b/server/server.go @@ -39,6 +39,7 @@ import ( "golang.org/x/sync/errgroup" + "github.com/beevik/ntp" "github.com/pelletier/go-toml" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/boltdb" @@ -285,6 +286,16 @@ func (m *Command) SetupServer() error { m.logger.Printf("%s", pilosa.VersionInfo()) + if pilosa.TrialDeadline != "" { + layout := "2006-02-02" + pilosa.TrialDeadline = "2021-02-06" + endTime, err := time.Parse(layout, pilosa.TrialDeadline) + if err != nil { + return errors.Wrap(err, "parsing curTime from make file") + } + go m.dailyCheck(endTime) + } + // If the pilosa command line uses -tx to override the // PILOSA_TXSRC env variable, then we must also correct // the environment, so that pilosa/txfactory.go can determine the @@ -478,6 +489,48 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "new handler") } +// dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days +func (m *Command) dailyCheck(end time.Time) error { + ticker := time.NewTicker(5 * time.Second) + var err error + for range ticker.C { + cur, err := m.ntpServerTime() + if err != nil { + return errors.Wrap(err, "reading ntp server time") + } + + m.logger.Printf("Current time remaining in trial: %v", end.Sub(cur)) + if end.Sub(cur) <= 0 { + m.logger.Printf("end: %v", end) // test line + m.logger.Printf("cur: %v", cur) // test line + m.logger.Printf("Your free trial has ended") + os.Exit(0) //is 0 the right exit number? + } + } + return errors.Wrap(err, "reading ntp server time") +} + +// ntpServerTime attempts to reach ntp servers with delays between each attempt +func (m *Command) ntpServerTime() (time.Time, error) { + curTime, err := ntp.Time("0.beevik-ntp.pool.ntp.org") + errCount := 0 + if err != nil { + for i := 0; i < 4; i++ { + curTime, err = ntp.Time("0.beevik-ntp.pool.ntp.org") + if err != nil { + errCount++ + if errCount >= 4 { + return curTime, err + } + time.Sleep(100 * time.Millisecond) + } else { + break + } + } + } + return curTime, err +} + // setupNetworking sets up internode communication based on the configuration. func (m *Command) setupNetworking() error { if m.Config.Cluster.Disabled { diff --git a/version.go b/version.go index 91a5841b7..7c3f2361b 100644 --- a/version.go +++ b/version.go @@ -25,6 +25,7 @@ var Variant string var BuildTime string var LatticeCommit string var GoVersion string = runtime.Version() +var TrialDeadline string func VersionInfo() string { var prefix string @@ -52,7 +53,7 @@ func VersionInfo() string { case buildTime != "": suffix += " (" + buildTime + ")" } - suffix += " " + GoVersion + suffix += " " + GoVersion + " limited time trial ends at: " + TrialDeadline return prefix + "Pilosa" + suffix } From fb47cb875a620cb466ee418fc593ba834d2c8332 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 5 Feb 2021 13:40:57 -0600 Subject: [PATCH 02/29] remove error return type --- server/server.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/server.go b/server/server.go index 1e9a8d053..7ce26a8d0 100644 --- a/server/server.go +++ b/server/server.go @@ -490,13 +490,14 @@ func (m *Command) SetupServer() error { } // dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days -func (m *Command) dailyCheck(end time.Time) error { +func (m *Command) dailyCheck(end time.Time) { ticker := time.NewTicker(5 * time.Second) var err error for range ticker.C { cur, err := m.ntpServerTime() if err != nil { - return errors.Wrap(err, "reading ntp server time") + errors.Wrap(err, "reading ntp server time") + os.Exit(1) } m.logger.Printf("Current time remaining in trial: %v", end.Sub(cur)) @@ -507,7 +508,8 @@ func (m *Command) dailyCheck(end time.Time) error { os.Exit(0) //is 0 the right exit number? } } - return errors.Wrap(err, "reading ntp server time") + errors.Wrap(err, "reading ntp server time") + os.Exit(1) } // ntpServerTime attempts to reach ntp servers with delays between each attempt From f37ec5b425f6596c8e2990ed14be6e16c9cf2c98 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 5 Feb 2021 14:25:57 -0600 Subject: [PATCH 03/29] alter error output to satisfy test --- server/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/server.go b/server/server.go index 7ce26a8d0..3e416436e 100644 --- a/server/server.go +++ b/server/server.go @@ -496,7 +496,7 @@ func (m *Command) dailyCheck(end time.Time) { for range ticker.C { cur, err := m.ntpServerTime() if err != nil { - errors.Wrap(err, "reading ntp server time") + m.logger.Printf("reading ntp server time %v", err) os.Exit(1) } @@ -508,7 +508,7 @@ func (m *Command) dailyCheck(end time.Time) { os.Exit(0) //is 0 the right exit number? } } - errors.Wrap(err, "reading ntp server time") + m.logger.Printf("reading ntp server time %v", err) os.Exit(1) } From c967bc96f4c5c782b1300635593e8009eefb2010 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 5 Feb 2021 15:40:55 -0600 Subject: [PATCH 04/29] remove test code --- server/server.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/server/server.go b/server/server.go index 3e416436e..089d29dba 100644 --- a/server/server.go +++ b/server/server.go @@ -287,8 +287,7 @@ func (m *Command) SetupServer() error { m.logger.Printf("%s", pilosa.VersionInfo()) if pilosa.TrialDeadline != "" { - layout := "2006-02-02" - pilosa.TrialDeadline = "2021-02-06" + layout := "2006-01-02" endTime, err := time.Parse(layout, pilosa.TrialDeadline) if err != nil { return errors.Wrap(err, "parsing curTime from make file") @@ -499,13 +498,10 @@ func (m *Command) dailyCheck(end time.Time) { m.logger.Printf("reading ntp server time %v", err) os.Exit(1) } - m.logger.Printf("Current time remaining in trial: %v", end.Sub(cur)) if end.Sub(cur) <= 0 { - m.logger.Printf("end: %v", end) // test line - m.logger.Printf("cur: %v", cur) // test line m.logger.Printf("Your free trial has ended") - os.Exit(0) //is 0 the right exit number? + os.Exit(0) } } m.logger.Printf("reading ntp server time %v", err) From 0227d8306fd7f42214f4f9fb35f2629c0734a58c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 9 Feb 2021 15:00:11 -0600 Subject: [PATCH 05/29] Upgrade lattice --- lattice | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lattice b/lattice index 2f0302c1d..fa773628a 160000 --- a/lattice +++ b/lattice @@ -1 +1 @@ -Subproject commit 2f0302c1d124433f0e1af5ae6c3bb7e4a64ca520 +Subproject commit fa773628a276e2590785a87fbc236c7e88ea6284 From 7ae1599c12e777405b8b0940abb6ded7056f6dd9 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Tue, 9 Feb 2021 15:39:32 -0600 Subject: [PATCH 06/29] edit and move trial code from server.go to trial.go and combine release build targets --- Makefile | 16 ++++--------- server/server.go | 51 +--------------------------------------- server/trial.go | 61 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 61 deletions(-) create mode 100644 server/trial.go diff --git a/Makefile b/Makefile index 8a724c282..e66f44249 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT) -X github.com/pilosa/pilosa/v2.TrialDeadline=$(TRIAL_DEADLINE)" +TRIAL_STRING = $(if $(TRIAL_DEADLINE),"-trial-$(TRIAL_DEADLINE)","") GO_VERSION=1.14.10 RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) @@ -98,19 +99,12 @@ cover-viz: cover build: go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa -# Create a single release trial build under the build directory -release-build-trial: - $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-trial-$(VERSION_ID)/pilosa" RELEASE=1 - cp NOTICE README.md build/pilosa-trial-$(VERSION_ID) - tar -cvz -C build -f build/pilosa-trial-$(VERSION_ID).tar.gz pilosa-trial-$(VERSION_ID)/ - @echo Created release trial build: build/pilosa-trial-$(VERSION_ID).tar.gz - # Create a single release build under the build directory release-build: - $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1 - cp NOTICE README.md LICENSE build/pilosa-$(VERSION_ID) - tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ - @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz + $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa$(TRIAL_STRING)-$(VERSION_ID)/pilosa" RELEASE=1 + cp NOTICE README.md LICENSE build/pilosa$(TRIAL_STRING)-$(VERSION_ID) + tar -cvz -C build -f build/pilosa$(TRIAL_STRING)-$(VERSION_ID).tar.gz pilosa$(TRIAL_STRING)-$(VERSION_ID)/ + @echo Created release build: build/pilosa$(TRIAL_STRING)-$(VERSION_ID).tar.gz # Error out if there are untracked changes in Git check-clean: diff --git a/server/server.go b/server/server.go index 089d29dba..cbd70a99f 100644 --- a/server/server.go +++ b/server/server.go @@ -39,7 +39,6 @@ import ( "golang.org/x/sync/errgroup" - "github.com/beevik/ntp" "github.com/pelletier/go-toml" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/boltdb" @@ -286,14 +285,7 @@ func (m *Command) SetupServer() error { m.logger.Printf("%s", pilosa.VersionInfo()) - if pilosa.TrialDeadline != "" { - layout := "2006-01-02" - endTime, err := time.Parse(layout, pilosa.TrialDeadline) - if err != nil { - return errors.Wrap(err, "parsing curTime from make file") - } - go m.dailyCheck(endTime) - } + m.trialVersion() // If the pilosa command line uses -tx to override the // PILOSA_TXSRC env variable, then we must also correct @@ -488,47 +480,6 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "new handler") } -// dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days -func (m *Command) dailyCheck(end time.Time) { - ticker := time.NewTicker(5 * time.Second) - var err error - for range ticker.C { - cur, err := m.ntpServerTime() - if err != nil { - m.logger.Printf("reading ntp server time %v", err) - os.Exit(1) - } - m.logger.Printf("Current time remaining in trial: %v", end.Sub(cur)) - if end.Sub(cur) <= 0 { - m.logger.Printf("Your free trial has ended") - os.Exit(0) - } - } - m.logger.Printf("reading ntp server time %v", err) - os.Exit(1) -} - -// ntpServerTime attempts to reach ntp servers with delays between each attempt -func (m *Command) ntpServerTime() (time.Time, error) { - curTime, err := ntp.Time("0.beevik-ntp.pool.ntp.org") - errCount := 0 - if err != nil { - for i := 0; i < 4; i++ { - curTime, err = ntp.Time("0.beevik-ntp.pool.ntp.org") - if err != nil { - errCount++ - if errCount >= 4 { - return curTime, err - } - time.Sleep(100 * time.Millisecond) - } else { - break - } - } - } - return curTime, err -} - // setupNetworking sets up internode communication based on the configuration. func (m *Command) setupNetworking() error { if m.Config.Cluster.Disabled { diff --git a/server/trial.go b/server/trial.go new file mode 100644 index 000000000..85df72422 --- /dev/null +++ b/server/trial.go @@ -0,0 +1,61 @@ +package server + +import ( + "os" + "time" + + "github.com/beevik/ntp" + "github.com/pilosa/pilosa/v2" +) + +func (m *Command) trialVersion() { + if pilosa.TrialDeadline != "" { + endTime, err := time.Parse("2006-01-02", pilosa.TrialDeadline) + if err != nil { + m.logger.Printf("parsing curTime from make file: %v", err) + os.Exit(1) + } + go m.dailyCheck(endTime) + } +} + +const hoursPerCheck = 24 * time.Hour + +// dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days +func (m *Command) dailyCheck(endTime time.Time) { + ticker := time.NewTicker(hoursPerCheck) + startTime, err := m.ntpServerTime(4) + if err != nil { + m.logger.Printf("reading ntp server time %v", err) + os.Exit(1) + } + runDuration := endTime.Sub(startTime) + if runDuration <= 0 { + m.logger.Printf("Trial edition of Molecula has expired, exiting now!") + os.Exit(1) + } + for range ticker.C { + runningDuration := time.Since(startTime) + m.logger.Printf("Current time remaining in trial: %v", runDuration-runningDuration) + if runningDuration >= runDuration { + m.logger.Printf("Trial edition of Molecula has expired, exiting now!") + os.Exit(1) + } + } +} + +const url = "0.beevik-ntp.pool.ntp.org" +const timeRequestDelay = 100 * time.Millisecond + +// ntpServerTime attempts to reach ntp servers with delays between each attempt, returning the time value of the first connected server +func (m *Command) ntpServerTime(retries int) (time.Time, error) { + t, err := ntp.Time(url) + if err != nil && retries <= 0 { + return t, err + } + if err != nil { + time.Sleep(timeRequestDelay) + return m.ntpServerTime(retries - 1) + } + return t, nil +} From 4bcdfcc9a2b81ad7d57cb9153eb50e196db4386e Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Wed, 10 Feb 2021 08:21:36 -0600 Subject: [PATCH 07/29] remove remaining release-build-trial --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e66f44249..0f06e1b92 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice release-build-trial +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice CLONE_URL=github.com/pilosa/pilosa MOD_VERSION=v2 From 302093035edefe8011ab736c86a53d5777983e7e Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Wed, 10 Feb 2021 12:37:13 -0600 Subject: [PATCH 08/29] add licence header --- server/trial.go | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/server/trial.go b/server/trial.go index 85df72422..8feb6bfa4 100644 --- a/server/trial.go +++ b/server/trial.go @@ -1,3 +1,22 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Package server contains the `pilosa server` subcommand which runs Pilosa +// itself. The purpose of this package is to define an easily tested Command +// object which handles interpreting configuration and setting up all the +// objects that Pilosa needs. + package server import ( @@ -12,18 +31,18 @@ func (m *Command) trialVersion() { if pilosa.TrialDeadline != "" { endTime, err := time.Parse("2006-01-02", pilosa.TrialDeadline) if err != nil { - m.logger.Printf("parsing curTime from make file: %v", err) + m.logger.Printf("parsing trial deadline: %v", err) os.Exit(1) } go m.dailyCheck(endTime) } } -const hoursPerCheck = 24 * time.Hour +const trialCheckInterval = 24 * time.Hour // dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days func (m *Command) dailyCheck(endTime time.Time) { - ticker := time.NewTicker(hoursPerCheck) + ticker := time.NewTicker(trialCheckInterval) startTime, err := m.ntpServerTime(4) if err != nil { m.logger.Printf("reading ntp server time %v", err) @@ -44,17 +63,17 @@ func (m *Command) dailyCheck(endTime time.Time) { } } -const url = "0.beevik-ntp.pool.ntp.org" -const timeRequestDelay = 100 * time.Millisecond +const ntpurl = "0.beevik-ntp.pool.ntp.org" +const ntpRetryDelay = 100 * time.Millisecond // ntpServerTime attempts to reach ntp servers with delays between each attempt, returning the time value of the first connected server func (m *Command) ntpServerTime(retries int) (time.Time, error) { - t, err := ntp.Time(url) + t, err := ntp.Time(ntpurl) if err != nil && retries <= 0 { return t, err } if err != nil { - time.Sleep(timeRequestDelay) + time.Sleep(ntpRetryDelay) return m.ntpServerTime(retries - 1) } return t, nil From e7f272f37f707f2462655182e5c12a9a8aac7886 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 10 Feb 2021 12:43:27 -0600 Subject: [PATCH 09/29] updates existence field on importroaring fixes issue (1411) --- api.go | 13 ++++++++++++- executor.go | 2 -- http/client_test.go | 46 +++++++++++++++++++++++++++++++++++++++++++++ http/handler.go | 1 - index.go | 1 - 5 files changed, 58 insertions(+), 5 deletions(-) diff --git a/api.go b/api.go index d2c09fe85..79246c271 100644 --- a/api.go +++ b/api.go @@ -416,6 +416,12 @@ func importWorker(importWork chan importJob) { case RequestActionSet: fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) if fileMagic == roaring.MagicNumber { // if pilosa roaring format + if ef := j.field.idx.existenceField(); ef != nil { + err = ef.importRoaring(j.ctx, tx, viewData, j.shard, "standard", false) + if err != nil { + return errors.Wrap(err, "importing pilosa roaring existence") + } + } err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear) if err != nil { return errors.Wrap(err, "importing pilosa roaring") @@ -425,6 +431,12 @@ func importWorker(importWork chan importJob) { // field.importRoaring changes the standard roaring run format to pilosa roaring data := make([]byte, len(viewData)) copy(data, viewData) + if ef := j.field.idx.existenceField(); ef != nil { + err = ef.importRoaring(j.ctx, tx, data, j.shard, "standard", false) + if err != nil { + return errors.Wrap(err, "importing pilosa roaring existence") + } + } err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) if err != nil { @@ -468,7 +480,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, span, ctx := tracing.StartSpanFromContext(ctx, "API.ImportRoaring") span.LogKV("index", indexName, "field", fieldName) defer span.Finish() - if err := api.validate(apiField); err != nil { return errors.Wrap(err, "validating api method") } diff --git a/executor.go b/executor.go index 6a50d4c2c..4b88d4270 100644 --- a/executor.go +++ b/executor.go @@ -4569,7 +4569,6 @@ func (e *executor) executeUnionRows(ctx context.Context, qcx *Qcx, index string, // executeAllCallShard executes an All() call for a local shard. func (e *executor) executeAllCallShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (res *Row, err0 error) { - span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeAllCallShard") defer span.Finish() @@ -4596,7 +4595,6 @@ func (e *executor) executeAllCallShard(ctx context.Context, qcx *Qcx, index stri } defer finisher(&err0) - if existenceRow, err = existenceFrag.row(tx, 0); err != nil { return nil, err } diff --git a/http/client_test.go b/http/client_test.go index b1b154647..4d2c7f315 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -1435,3 +1435,49 @@ func TestClient_ServerInfoHasTxSrc(t *testing.T) { } pilosa.MustTxsrcToTxtype(si.TxSrc) // panics if invalid } +func TestClient_ImportRoaringExists(t *testing.T) { + cluster := test.MustNewCluster(t, 1) + err := cluster.Start() + if err != nil { + t.Fatalf("starting cluster: %v", err) + } + defer cluster.Close() + + node := cluster.GetNode(0) + _, err = node.API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = node.API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + // Send import request. + host := node.URL() + c := MustNewClient(host, http.GetHTTPClient(nil)) + // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] + roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") + + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { + t.Fatal(err) + } + expected := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537} + var qr pilosa.QueryResponse + qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Row(f=0)"}) + if err != nil { + t.Fatalf("%v", err) + } + got := qr.Results[0].(*pilosa.Row).Columns() + if !reflect.DeepEqual(got, expected) { + t.Fatalf(" Row unexpected columns: got %+v expected: %+v", got, expected) + } + qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + if err != nil { + t.Fatalf("%v", err) + } + got = qr.Results[0].(*pilosa.Row).Columns() + if !reflect.DeepEqual(got, expected) { + t.Fatalf("All unexpected columns: got %+v expected: %+v", got, expected) + } + +} diff --git a/http/handler.go b/http/handler.go index a6dfd5828..b7e44f67e 100644 --- a/http/handler.go +++ b/http/handler.go @@ -2486,7 +2486,6 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request http.Error(w, error, code) return } - // Get index and field type to determine how to handle the // import data. indexName := mux.Vars(r)["index"] diff --git a/index.go b/index.go index 6129289f5..94f47c6c7 100644 --- a/index.go +++ b/index.go @@ -482,7 +482,6 @@ func (i *Index) Fields() []*Field { func (i *Index) existenceField() *Field { i.mu.RLock() defer i.mu.RUnlock() - return i.existenceFld } From a61ed011fc1c4fd0ee3553e0350d7abb08ad9c53 Mon Sep 17 00:00:00 2001 From: tgruben Date: Wed, 10 Feb 2021 14:51:11 -0600 Subject: [PATCH 10/29] Revert "Update existence field on import-roaring requests" --- api.go | 13 +------------ executor.go | 2 ++ http/client_test.go | 46 --------------------------------------------- http/handler.go | 1 + index.go | 1 + 5 files changed, 5 insertions(+), 58 deletions(-) diff --git a/api.go b/api.go index 79246c271..d2c09fe85 100644 --- a/api.go +++ b/api.go @@ -416,12 +416,6 @@ func importWorker(importWork chan importJob) { case RequestActionSet: fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) if fileMagic == roaring.MagicNumber { // if pilosa roaring format - if ef := j.field.idx.existenceField(); ef != nil { - err = ef.importRoaring(j.ctx, tx, viewData, j.shard, "standard", false) - if err != nil { - return errors.Wrap(err, "importing pilosa roaring existence") - } - } err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear) if err != nil { return errors.Wrap(err, "importing pilosa roaring") @@ -431,12 +425,6 @@ func importWorker(importWork chan importJob) { // field.importRoaring changes the standard roaring run format to pilosa roaring data := make([]byte, len(viewData)) copy(data, viewData) - if ef := j.field.idx.existenceField(); ef != nil { - err = ef.importRoaring(j.ctx, tx, data, j.shard, "standard", false) - if err != nil { - return errors.Wrap(err, "importing pilosa roaring existence") - } - } err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) if err != nil { @@ -480,6 +468,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, span, ctx := tracing.StartSpanFromContext(ctx, "API.ImportRoaring") span.LogKV("index", indexName, "field", fieldName) defer span.Finish() + if err := api.validate(apiField); err != nil { return errors.Wrap(err, "validating api method") } diff --git a/executor.go b/executor.go index 4b88d4270..6a50d4c2c 100644 --- a/executor.go +++ b/executor.go @@ -4569,6 +4569,7 @@ func (e *executor) executeUnionRows(ctx context.Context, qcx *Qcx, index string, // executeAllCallShard executes an All() call for a local shard. func (e *executor) executeAllCallShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (res *Row, err0 error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeAllCallShard") defer span.Finish() @@ -4595,6 +4596,7 @@ func (e *executor) executeAllCallShard(ctx context.Context, qcx *Qcx, index stri } defer finisher(&err0) + if existenceRow, err = existenceFrag.row(tx, 0); err != nil { return nil, err } diff --git a/http/client_test.go b/http/client_test.go index 4d2c7f315..b1b154647 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -1435,49 +1435,3 @@ func TestClient_ServerInfoHasTxSrc(t *testing.T) { } pilosa.MustTxsrcToTxtype(si.TxSrc) // panics if invalid } -func TestClient_ImportRoaringExists(t *testing.T) { - cluster := test.MustNewCluster(t, 1) - err := cluster.Start() - if err != nil { - t.Fatalf("starting cluster: %v", err) - } - defer cluster.Close() - - node := cluster.GetNode(0) - _, err = node.API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{TrackExistence: true}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - _, err = node.API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) - if err != nil { - t.Fatalf("creating field: %v", err) - } - // Send import request. - host := node.URL() - c := MustNewClient(host, http.GetHTTPClient(nil)) - // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] - roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") - - if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { - t.Fatal(err) - } - expected := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537} - var qr pilosa.QueryResponse - qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Row(f=0)"}) - if err != nil { - t.Fatalf("%v", err) - } - got := qr.Results[0].(*pilosa.Row).Columns() - if !reflect.DeepEqual(got, expected) { - t.Fatalf(" Row unexpected columns: got %+v expected: %+v", got, expected) - } - qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) - if err != nil { - t.Fatalf("%v", err) - } - got = qr.Results[0].(*pilosa.Row).Columns() - if !reflect.DeepEqual(got, expected) { - t.Fatalf("All unexpected columns: got %+v expected: %+v", got, expected) - } - -} diff --git a/http/handler.go b/http/handler.go index b7e44f67e..a6dfd5828 100644 --- a/http/handler.go +++ b/http/handler.go @@ -2486,6 +2486,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request http.Error(w, error, code) return } + // Get index and field type to determine how to handle the // import data. indexName := mux.Vars(r)["index"] diff --git a/index.go b/index.go index 94f47c6c7..6129289f5 100644 --- a/index.go +++ b/index.go @@ -482,6 +482,7 @@ func (i *Index) Fields() []*Field { func (i *Index) existenceField() *Field { i.mu.RLock() defer i.mu.RUnlock() + return i.existenceFld } From 0f952bb5948ac289af168160729f8632996e6132 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Thu, 11 Feb 2021 10:36:04 -0600 Subject: [PATCH 11/29] Respond to code review feedback change logic in version.go so that the trial related messages only appear on trial versions of molecula convert Command methods in trial.go to functions and pass a loggerLogger variable instead since that was the only piece of Command being used add a function named expireAfter which seperately runs similar functionality to what was previously in daily check with chnages directed at stopping users from changing their internal clock date change variable names and placement to be more readable and organized --- server/server.go | 2 +- server/trial.go | 54 +++++++++++++++++++++++++----------------------- version.go | 6 ++++-- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/server/server.go b/server/server.go index cbd70a99f..b68829c4b 100644 --- a/server/server.go +++ b/server/server.go @@ -285,7 +285,7 @@ func (m *Command) SetupServer() error { m.logger.Printf("%s", pilosa.VersionInfo()) - m.trialVersion() + trialVersion(m.logger) // If the pilosa command line uses -tx to override the // PILOSA_TXSRC env variable, then we must also correct diff --git a/server/trial.go b/server/trial.go index 8feb6bfa4..868a177c5 100644 --- a/server/trial.go +++ b/server/trial.go @@ -1,4 +1,4 @@ -// Copyright 2017 Pilosa Corp. +// Copyright 2021 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -27,54 +27,56 @@ import ( "github.com/pilosa/pilosa/v2" ) -func (m *Command) trialVersion() { +func trialVersion(logger loggerLogger) { if pilosa.TrialDeadline != "" { - endTime, err := time.Parse("2006-01-02", pilosa.TrialDeadline) + startTime, err := ntpServerTime(4, logger) if err != nil { - m.logger.Printf("parsing trial deadline: %v", err) + logger.Printf("reading ntp server time %v", err) os.Exit(1) } - go m.dailyCheck(endTime) + endTime, err := time.Parse("2006-01-02", pilosa.TrialDeadline) + if err != nil { + logger.Printf("parsing trial deadline: %v", err) + os.Exit(1) + } + maxDuration := endTime.Sub(startTime) + go expireAfter(maxDuration, logger) + go dailyCheck(maxDuration, logger) } } const trialCheckInterval = 24 * time.Hour // dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days -func (m *Command) dailyCheck(endTime time.Time) { +func dailyCheck(maxDuration time.Duration, logger loggerLogger) { + startTime := time.Now() // we get a new start time here to ensure that it has a monotonic clock + remaining := maxDuration - time.Since(startTime) + logger.Printf("Current time remaining in trial: %d days %v", remaining/(time.Hour*24), remaining%(time.Hour*24)) ticker := time.NewTicker(trialCheckInterval) - startTime, err := m.ntpServerTime(4) - if err != nil { - m.logger.Printf("reading ntp server time %v", err) - os.Exit(1) - } - runDuration := endTime.Sub(startTime) - if runDuration <= 0 { - m.logger.Printf("Trial edition of Molecula has expired, exiting now!") - os.Exit(1) - } for range ticker.C { - runningDuration := time.Since(startTime) - m.logger.Printf("Current time remaining in trial: %v", runDuration-runningDuration) - if runningDuration >= runDuration { - m.logger.Printf("Trial edition of Molecula has expired, exiting now!") - os.Exit(1) - } + remaining := maxDuration - time.Since(startTime) + logger.Printf("Current time remaining in trial: %d days %v", remaining/(time.Hour*24), remaining%(time.Hour*24)) } } -const ntpurl = "0.beevik-ntp.pool.ntp.org" +const ntpURL = "0.beevik-ntp.pool.ntp.org" const ntpRetryDelay = 100 * time.Millisecond // ntpServerTime attempts to reach ntp servers with delays between each attempt, returning the time value of the first connected server -func (m *Command) ntpServerTime(retries int) (time.Time, error) { - t, err := ntp.Time(ntpurl) +func ntpServerTime(retries int, logger loggerLogger) (time.Time, error) { + t, err := ntp.Time(ntpURL) if err != nil && retries <= 0 { return t, err } if err != nil { time.Sleep(ntpRetryDelay) - return m.ntpServerTime(retries - 1) + return ntpServerTime(retries-1, logger) } return t, nil } + +func expireAfter(maxDuration time.Duration, logger loggerLogger) { + time.Sleep(maxDuration) + logger.Printf("Trial edition of Molecula has expired, exiting now!") + os.Exit(1) +} diff --git a/version.go b/version.go index 7c3f2361b..ec27e5c6f 100644 --- a/version.go +++ b/version.go @@ -53,8 +53,10 @@ func VersionInfo() string { case buildTime != "": suffix += " (" + buildTime + ")" } - suffix += " " + GoVersion + " limited time trial ends at: " + TrialDeadline - + suffix += " " + GoVersion + if TrialDeadline != "" { + suffix += " limited time trial ends at: " + TrialDeadline + } return prefix + "Pilosa" + suffix } From d357447673b5802cab9b4feabc325edbedf8c12b Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Thu, 11 Feb 2021 10:59:32 -0600 Subject: [PATCH 12/29] change wording --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index ec27e5c6f..73a78f5c7 100644 --- a/version.go +++ b/version.go @@ -55,7 +55,7 @@ func VersionInfo() string { } suffix += " " + GoVersion if TrialDeadline != "" { - suffix += " limited time trial ends at: " + TrialDeadline + suffix += " limited time trial ends on: " + TrialDeadline } return prefix + "Pilosa" + suffix } From 1dbab1ff92b9b62c067e9182f47d660bd8b7ed13 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 5 Feb 2021 12:16:08 -0600 Subject: [PATCH 13/29] add draft trial version of molecula --- Makefile | 11 ++++++++-- go.mod | 1 + go.sum | 2 ++ server/server.go | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ version.go | 3 ++- 5 files changed, 67 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 2e6cc7331..8a724c282 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice release-build-trial CLONE_URL=github.com/pilosa/pilosa MOD_VERSION=v2 @@ -11,7 +11,7 @@ BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) -LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)" +LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT) -X github.com/pilosa/pilosa/v2.TrialDeadline=$(TRIAL_DEADLINE)" GO_VERSION=1.14.10 RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) @@ -98,6 +98,13 @@ cover-viz: cover build: go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa +# Create a single release trial build under the build directory +release-build-trial: + $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-trial-$(VERSION_ID)/pilosa" RELEASE=1 + cp NOTICE README.md build/pilosa-trial-$(VERSION_ID) + tar -cvz -C build -f build/pilosa-trial-$(VERSION_ID).tar.gz pilosa-trial-$(VERSION_ID)/ + @echo Created release trial build: build/pilosa-trial-$(VERSION_ID).tar.gz + # Create a single release build under the build directory release-build: $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1 diff --git a/go.mod b/go.mod index f0d161c95..dcd6b603a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 + github.com/beevik/ntp v0.3.0 github.com/benbjohnson/immutable v0.3.0 github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect diff --git a/go.sum b/go.sum index b9d5e44d7..f1ad4ecc5 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= +github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/immutable v0.3.0 h1:TVRhuZx2wG9SZ0LRdqlbs9S5BZ6Y24hJEHTCgWHZEIw= github.com/benbjohnson/immutable v0.3.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= diff --git a/server/server.go b/server/server.go index 530e4102a..1e9a8d053 100644 --- a/server/server.go +++ b/server/server.go @@ -39,6 +39,7 @@ import ( "golang.org/x/sync/errgroup" + "github.com/beevik/ntp" "github.com/pelletier/go-toml" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/boltdb" @@ -285,6 +286,16 @@ func (m *Command) SetupServer() error { m.logger.Printf("%s", pilosa.VersionInfo()) + if pilosa.TrialDeadline != "" { + layout := "2006-02-02" + pilosa.TrialDeadline = "2021-02-06" + endTime, err := time.Parse(layout, pilosa.TrialDeadline) + if err != nil { + return errors.Wrap(err, "parsing curTime from make file") + } + go m.dailyCheck(endTime) + } + // If the pilosa command line uses -tx to override the // PILOSA_TXSRC env variable, then we must also correct // the environment, so that pilosa/txfactory.go can determine the @@ -478,6 +489,48 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "new handler") } +// dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days +func (m *Command) dailyCheck(end time.Time) error { + ticker := time.NewTicker(5 * time.Second) + var err error + for range ticker.C { + cur, err := m.ntpServerTime() + if err != nil { + return errors.Wrap(err, "reading ntp server time") + } + + m.logger.Printf("Current time remaining in trial: %v", end.Sub(cur)) + if end.Sub(cur) <= 0 { + m.logger.Printf("end: %v", end) // test line + m.logger.Printf("cur: %v", cur) // test line + m.logger.Printf("Your free trial has ended") + os.Exit(0) //is 0 the right exit number? + } + } + return errors.Wrap(err, "reading ntp server time") +} + +// ntpServerTime attempts to reach ntp servers with delays between each attempt +func (m *Command) ntpServerTime() (time.Time, error) { + curTime, err := ntp.Time("0.beevik-ntp.pool.ntp.org") + errCount := 0 + if err != nil { + for i := 0; i < 4; i++ { + curTime, err = ntp.Time("0.beevik-ntp.pool.ntp.org") + if err != nil { + errCount++ + if errCount >= 4 { + return curTime, err + } + time.Sleep(100 * time.Millisecond) + } else { + break + } + } + } + return curTime, err +} + // setupNetworking sets up internode communication based on the configuration. func (m *Command) setupNetworking() error { if m.Config.Cluster.Disabled { diff --git a/version.go b/version.go index 91a5841b7..7c3f2361b 100644 --- a/version.go +++ b/version.go @@ -25,6 +25,7 @@ var Variant string var BuildTime string var LatticeCommit string var GoVersion string = runtime.Version() +var TrialDeadline string func VersionInfo() string { var prefix string @@ -52,7 +53,7 @@ func VersionInfo() string { case buildTime != "": suffix += " (" + buildTime + ")" } - suffix += " " + GoVersion + suffix += " " + GoVersion + " limited time trial ends at: " + TrialDeadline return prefix + "Pilosa" + suffix } From 9840ae67e4fa43ce0912038a13b81c91b2fee8da Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 5 Feb 2021 13:40:57 -0600 Subject: [PATCH 14/29] remove error return type --- server/server.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/server.go b/server/server.go index 1e9a8d053..7ce26a8d0 100644 --- a/server/server.go +++ b/server/server.go @@ -490,13 +490,14 @@ func (m *Command) SetupServer() error { } // dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days -func (m *Command) dailyCheck(end time.Time) error { +func (m *Command) dailyCheck(end time.Time) { ticker := time.NewTicker(5 * time.Second) var err error for range ticker.C { cur, err := m.ntpServerTime() if err != nil { - return errors.Wrap(err, "reading ntp server time") + errors.Wrap(err, "reading ntp server time") + os.Exit(1) } m.logger.Printf("Current time remaining in trial: %v", end.Sub(cur)) @@ -507,7 +508,8 @@ func (m *Command) dailyCheck(end time.Time) error { os.Exit(0) //is 0 the right exit number? } } - return errors.Wrap(err, "reading ntp server time") + errors.Wrap(err, "reading ntp server time") + os.Exit(1) } // ntpServerTime attempts to reach ntp servers with delays between each attempt From d02ba3274340c07b45d3169b52a98a7fb08042c8 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 5 Feb 2021 14:25:57 -0600 Subject: [PATCH 15/29] alter error output to satisfy test --- server/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/server.go b/server/server.go index 7ce26a8d0..3e416436e 100644 --- a/server/server.go +++ b/server/server.go @@ -496,7 +496,7 @@ func (m *Command) dailyCheck(end time.Time) { for range ticker.C { cur, err := m.ntpServerTime() if err != nil { - errors.Wrap(err, "reading ntp server time") + m.logger.Printf("reading ntp server time %v", err) os.Exit(1) } @@ -508,7 +508,7 @@ func (m *Command) dailyCheck(end time.Time) { os.Exit(0) //is 0 the right exit number? } } - errors.Wrap(err, "reading ntp server time") + m.logger.Printf("reading ntp server time %v", err) os.Exit(1) } From 7fd3fc775fb0d7e3d99458b67d5746d9ac1235fc Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 5 Feb 2021 15:40:55 -0600 Subject: [PATCH 16/29] remove test code --- server/server.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/server/server.go b/server/server.go index 3e416436e..089d29dba 100644 --- a/server/server.go +++ b/server/server.go @@ -287,8 +287,7 @@ func (m *Command) SetupServer() error { m.logger.Printf("%s", pilosa.VersionInfo()) if pilosa.TrialDeadline != "" { - layout := "2006-02-02" - pilosa.TrialDeadline = "2021-02-06" + layout := "2006-01-02" endTime, err := time.Parse(layout, pilosa.TrialDeadline) if err != nil { return errors.Wrap(err, "parsing curTime from make file") @@ -499,13 +498,10 @@ func (m *Command) dailyCheck(end time.Time) { m.logger.Printf("reading ntp server time %v", err) os.Exit(1) } - m.logger.Printf("Current time remaining in trial: %v", end.Sub(cur)) if end.Sub(cur) <= 0 { - m.logger.Printf("end: %v", end) // test line - m.logger.Printf("cur: %v", cur) // test line m.logger.Printf("Your free trial has ended") - os.Exit(0) //is 0 the right exit number? + os.Exit(0) } } m.logger.Printf("reading ntp server time %v", err) From c60893a8eec2bab013a33a6b33e974c8d32c2c9b Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Tue, 9 Feb 2021 15:39:32 -0600 Subject: [PATCH 17/29] edit and move trial code from server.go to trial.go and combine release build targets --- Makefile | 16 ++++--------- server/server.go | 51 +--------------------------------------- server/trial.go | 61 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 61 deletions(-) create mode 100644 server/trial.go diff --git a/Makefile b/Makefile index 8a724c282..e66f44249 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT) -X github.com/pilosa/pilosa/v2.TrialDeadline=$(TRIAL_DEADLINE)" +TRIAL_STRING = $(if $(TRIAL_DEADLINE),"-trial-$(TRIAL_DEADLINE)","") GO_VERSION=1.14.10 RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) @@ -98,19 +99,12 @@ cover-viz: cover build: go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa -# Create a single release trial build under the build directory -release-build-trial: - $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-trial-$(VERSION_ID)/pilosa" RELEASE=1 - cp NOTICE README.md build/pilosa-trial-$(VERSION_ID) - tar -cvz -C build -f build/pilosa-trial-$(VERSION_ID).tar.gz pilosa-trial-$(VERSION_ID)/ - @echo Created release trial build: build/pilosa-trial-$(VERSION_ID).tar.gz - # Create a single release build under the build directory release-build: - $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1 - cp NOTICE README.md LICENSE build/pilosa-$(VERSION_ID) - tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ - @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz + $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa$(TRIAL_STRING)-$(VERSION_ID)/pilosa" RELEASE=1 + cp NOTICE README.md LICENSE build/pilosa$(TRIAL_STRING)-$(VERSION_ID) + tar -cvz -C build -f build/pilosa$(TRIAL_STRING)-$(VERSION_ID).tar.gz pilosa$(TRIAL_STRING)-$(VERSION_ID)/ + @echo Created release build: build/pilosa$(TRIAL_STRING)-$(VERSION_ID).tar.gz # Error out if there are untracked changes in Git check-clean: diff --git a/server/server.go b/server/server.go index 089d29dba..cbd70a99f 100644 --- a/server/server.go +++ b/server/server.go @@ -39,7 +39,6 @@ import ( "golang.org/x/sync/errgroup" - "github.com/beevik/ntp" "github.com/pelletier/go-toml" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/boltdb" @@ -286,14 +285,7 @@ func (m *Command) SetupServer() error { m.logger.Printf("%s", pilosa.VersionInfo()) - if pilosa.TrialDeadline != "" { - layout := "2006-01-02" - endTime, err := time.Parse(layout, pilosa.TrialDeadline) - if err != nil { - return errors.Wrap(err, "parsing curTime from make file") - } - go m.dailyCheck(endTime) - } + m.trialVersion() // If the pilosa command line uses -tx to override the // PILOSA_TXSRC env variable, then we must also correct @@ -488,47 +480,6 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "new handler") } -// dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days -func (m *Command) dailyCheck(end time.Time) { - ticker := time.NewTicker(5 * time.Second) - var err error - for range ticker.C { - cur, err := m.ntpServerTime() - if err != nil { - m.logger.Printf("reading ntp server time %v", err) - os.Exit(1) - } - m.logger.Printf("Current time remaining in trial: %v", end.Sub(cur)) - if end.Sub(cur) <= 0 { - m.logger.Printf("Your free trial has ended") - os.Exit(0) - } - } - m.logger.Printf("reading ntp server time %v", err) - os.Exit(1) -} - -// ntpServerTime attempts to reach ntp servers with delays between each attempt -func (m *Command) ntpServerTime() (time.Time, error) { - curTime, err := ntp.Time("0.beevik-ntp.pool.ntp.org") - errCount := 0 - if err != nil { - for i := 0; i < 4; i++ { - curTime, err = ntp.Time("0.beevik-ntp.pool.ntp.org") - if err != nil { - errCount++ - if errCount >= 4 { - return curTime, err - } - time.Sleep(100 * time.Millisecond) - } else { - break - } - } - } - return curTime, err -} - // setupNetworking sets up internode communication based on the configuration. func (m *Command) setupNetworking() error { if m.Config.Cluster.Disabled { diff --git a/server/trial.go b/server/trial.go new file mode 100644 index 000000000..85df72422 --- /dev/null +++ b/server/trial.go @@ -0,0 +1,61 @@ +package server + +import ( + "os" + "time" + + "github.com/beevik/ntp" + "github.com/pilosa/pilosa/v2" +) + +func (m *Command) trialVersion() { + if pilosa.TrialDeadline != "" { + endTime, err := time.Parse("2006-01-02", pilosa.TrialDeadline) + if err != nil { + m.logger.Printf("parsing curTime from make file: %v", err) + os.Exit(1) + } + go m.dailyCheck(endTime) + } +} + +const hoursPerCheck = 24 * time.Hour + +// dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days +func (m *Command) dailyCheck(endTime time.Time) { + ticker := time.NewTicker(hoursPerCheck) + startTime, err := m.ntpServerTime(4) + if err != nil { + m.logger.Printf("reading ntp server time %v", err) + os.Exit(1) + } + runDuration := endTime.Sub(startTime) + if runDuration <= 0 { + m.logger.Printf("Trial edition of Molecula has expired, exiting now!") + os.Exit(1) + } + for range ticker.C { + runningDuration := time.Since(startTime) + m.logger.Printf("Current time remaining in trial: %v", runDuration-runningDuration) + if runningDuration >= runDuration { + m.logger.Printf("Trial edition of Molecula has expired, exiting now!") + os.Exit(1) + } + } +} + +const url = "0.beevik-ntp.pool.ntp.org" +const timeRequestDelay = 100 * time.Millisecond + +// ntpServerTime attempts to reach ntp servers with delays between each attempt, returning the time value of the first connected server +func (m *Command) ntpServerTime(retries int) (time.Time, error) { + t, err := ntp.Time(url) + if err != nil && retries <= 0 { + return t, err + } + if err != nil { + time.Sleep(timeRequestDelay) + return m.ntpServerTime(retries - 1) + } + return t, nil +} From c973fae6aa4192e858c960b681950b275ac3055e Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Wed, 10 Feb 2021 08:21:36 -0600 Subject: [PATCH 18/29] remove remaining release-build-trial --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e66f44249..0f06e1b92 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice release-build-trial +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice CLONE_URL=github.com/pilosa/pilosa MOD_VERSION=v2 From ddcf9fe1e8c360961d87cb4bfebe124105a40f8b Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Wed, 10 Feb 2021 12:37:13 -0600 Subject: [PATCH 19/29] add licence header --- server/trial.go | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/server/trial.go b/server/trial.go index 85df72422..8feb6bfa4 100644 --- a/server/trial.go +++ b/server/trial.go @@ -1,3 +1,22 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Package server contains the `pilosa server` subcommand which runs Pilosa +// itself. The purpose of this package is to define an easily tested Command +// object which handles interpreting configuration and setting up all the +// objects that Pilosa needs. + package server import ( @@ -12,18 +31,18 @@ func (m *Command) trialVersion() { if pilosa.TrialDeadline != "" { endTime, err := time.Parse("2006-01-02", pilosa.TrialDeadline) if err != nil { - m.logger.Printf("parsing curTime from make file: %v", err) + m.logger.Printf("parsing trial deadline: %v", err) os.Exit(1) } go m.dailyCheck(endTime) } } -const hoursPerCheck = 24 * time.Hour +const trialCheckInterval = 24 * time.Hour // dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days func (m *Command) dailyCheck(endTime time.Time) { - ticker := time.NewTicker(hoursPerCheck) + ticker := time.NewTicker(trialCheckInterval) startTime, err := m.ntpServerTime(4) if err != nil { m.logger.Printf("reading ntp server time %v", err) @@ -44,17 +63,17 @@ func (m *Command) dailyCheck(endTime time.Time) { } } -const url = "0.beevik-ntp.pool.ntp.org" -const timeRequestDelay = 100 * time.Millisecond +const ntpurl = "0.beevik-ntp.pool.ntp.org" +const ntpRetryDelay = 100 * time.Millisecond // ntpServerTime attempts to reach ntp servers with delays between each attempt, returning the time value of the first connected server func (m *Command) ntpServerTime(retries int) (time.Time, error) { - t, err := ntp.Time(url) + t, err := ntp.Time(ntpurl) if err != nil && retries <= 0 { return t, err } if err != nil { - time.Sleep(timeRequestDelay) + time.Sleep(ntpRetryDelay) return m.ntpServerTime(retries - 1) } return t, nil From dd8d6ebfb197c8ec88b68101abed46bceaa8ba49 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Thu, 11 Feb 2021 10:36:04 -0600 Subject: [PATCH 20/29] Respond to code review feedback change logic in version.go so that the trial related messages only appear on trial versions of molecula convert Command methods in trial.go to functions and pass a loggerLogger variable instead since that was the only piece of Command being used add a function named expireAfter which seperately runs similar functionality to what was previously in daily check with chnages directed at stopping users from changing their internal clock date change variable names and placement to be more readable and organized --- server/server.go | 2 +- server/trial.go | 54 +++++++++++++++++++++++++----------------------- version.go | 6 ++++-- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/server/server.go b/server/server.go index cbd70a99f..b68829c4b 100644 --- a/server/server.go +++ b/server/server.go @@ -285,7 +285,7 @@ func (m *Command) SetupServer() error { m.logger.Printf("%s", pilosa.VersionInfo()) - m.trialVersion() + trialVersion(m.logger) // If the pilosa command line uses -tx to override the // PILOSA_TXSRC env variable, then we must also correct diff --git a/server/trial.go b/server/trial.go index 8feb6bfa4..868a177c5 100644 --- a/server/trial.go +++ b/server/trial.go @@ -1,4 +1,4 @@ -// Copyright 2017 Pilosa Corp. +// Copyright 2021 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -27,54 +27,56 @@ import ( "github.com/pilosa/pilosa/v2" ) -func (m *Command) trialVersion() { +func trialVersion(logger loggerLogger) { if pilosa.TrialDeadline != "" { - endTime, err := time.Parse("2006-01-02", pilosa.TrialDeadline) + startTime, err := ntpServerTime(4, logger) if err != nil { - m.logger.Printf("parsing trial deadline: %v", err) + logger.Printf("reading ntp server time %v", err) os.Exit(1) } - go m.dailyCheck(endTime) + endTime, err := time.Parse("2006-01-02", pilosa.TrialDeadline) + if err != nil { + logger.Printf("parsing trial deadline: %v", err) + os.Exit(1) + } + maxDuration := endTime.Sub(startTime) + go expireAfter(maxDuration, logger) + go dailyCheck(maxDuration, logger) } } const trialCheckInterval = 24 * time.Hour // dailyCheck runs in the background while a trial version of Molecula is being run, displaying daily reminders of the remaining days -func (m *Command) dailyCheck(endTime time.Time) { +func dailyCheck(maxDuration time.Duration, logger loggerLogger) { + startTime := time.Now() // we get a new start time here to ensure that it has a monotonic clock + remaining := maxDuration - time.Since(startTime) + logger.Printf("Current time remaining in trial: %d days %v", remaining/(time.Hour*24), remaining%(time.Hour*24)) ticker := time.NewTicker(trialCheckInterval) - startTime, err := m.ntpServerTime(4) - if err != nil { - m.logger.Printf("reading ntp server time %v", err) - os.Exit(1) - } - runDuration := endTime.Sub(startTime) - if runDuration <= 0 { - m.logger.Printf("Trial edition of Molecula has expired, exiting now!") - os.Exit(1) - } for range ticker.C { - runningDuration := time.Since(startTime) - m.logger.Printf("Current time remaining in trial: %v", runDuration-runningDuration) - if runningDuration >= runDuration { - m.logger.Printf("Trial edition of Molecula has expired, exiting now!") - os.Exit(1) - } + remaining := maxDuration - time.Since(startTime) + logger.Printf("Current time remaining in trial: %d days %v", remaining/(time.Hour*24), remaining%(time.Hour*24)) } } -const ntpurl = "0.beevik-ntp.pool.ntp.org" +const ntpURL = "0.beevik-ntp.pool.ntp.org" const ntpRetryDelay = 100 * time.Millisecond // ntpServerTime attempts to reach ntp servers with delays between each attempt, returning the time value of the first connected server -func (m *Command) ntpServerTime(retries int) (time.Time, error) { - t, err := ntp.Time(ntpurl) +func ntpServerTime(retries int, logger loggerLogger) (time.Time, error) { + t, err := ntp.Time(ntpURL) if err != nil && retries <= 0 { return t, err } if err != nil { time.Sleep(ntpRetryDelay) - return m.ntpServerTime(retries - 1) + return ntpServerTime(retries-1, logger) } return t, nil } + +func expireAfter(maxDuration time.Duration, logger loggerLogger) { + time.Sleep(maxDuration) + logger.Printf("Trial edition of Molecula has expired, exiting now!") + os.Exit(1) +} diff --git a/version.go b/version.go index 7c3f2361b..ec27e5c6f 100644 --- a/version.go +++ b/version.go @@ -53,8 +53,10 @@ func VersionInfo() string { case buildTime != "": suffix += " (" + buildTime + ")" } - suffix += " " + GoVersion + " limited time trial ends at: " + TrialDeadline - + suffix += " " + GoVersion + if TrialDeadline != "" { + suffix += " limited time trial ends at: " + TrialDeadline + } return prefix + "Pilosa" + suffix } From 136a4b8068f56d1d9759fad8057ee71fe426276d Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Thu, 11 Feb 2021 10:59:32 -0600 Subject: [PATCH 21/29] change wording --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index ec27e5c6f..73a78f5c7 100644 --- a/version.go +++ b/version.go @@ -55,7 +55,7 @@ func VersionInfo() string { } suffix += " " + GoVersion if TrialDeadline != "" { - suffix += " limited time trial ends at: " + TrialDeadline + suffix += " limited time trial ends on: " + TrialDeadline } return prefix + "Pilosa" + suffix } From b89dae01cc130a7734060074e53e70582d36099b Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Thu, 11 Feb 2021 16:13:04 -0600 Subject: [PATCH 22/29] clarify purpose of trialVersion by changing the name to handleTrialDeadline and adding a doc string --- lattice | 2 +- server/server.go | 2 +- server/trial.go | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lattice b/lattice index fa773628a..28c2313ec 160000 --- a/lattice +++ b/lattice @@ -1 +1 @@ -Subproject commit fa773628a276e2590785a87fbc236c7e88ea6284 +Subproject commit 28c2313ecfcd7e083d42d4e409483e968b4c421b diff --git a/server/server.go b/server/server.go index b68829c4b..b5b43cbc5 100644 --- a/server/server.go +++ b/server/server.go @@ -285,7 +285,7 @@ func (m *Command) SetupServer() error { m.logger.Printf("%s", pilosa.VersionInfo()) - trialVersion(m.logger) + handleTrialDeadline(m.logger) // If the pilosa command line uses -tx to override the // PILOSA_TXSRC env variable, then we must also correct diff --git a/server/trial.go b/server/trial.go index 868a177c5..342d979f9 100644 --- a/server/trial.go +++ b/server/trial.go @@ -27,7 +27,11 @@ import ( "github.com/pilosa/pilosa/v2" ) -func trialVersion(logger loggerLogger) { +// handleTrialDeadline checks to see if this is a trial version of Molecula that expires at some point. +// If it is, we contact an NTP server to get the current time and compare that to the trial deadline. +// We launch two goroutines, one which reminds via a log message how much time is left in the trial, +// and one which causes the process to exit once the trial is over. +func handleTrialDeadline(logger loggerLogger) { if pilosa.TrialDeadline != "" { startTime, err := ntpServerTime(4, logger) if err != nil { From 56c940c6a0b7738a6dbf198c4cf0e38e3470b418 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 3 Feb 2021 19:38:15 -0600 Subject: [PATCH 23/29] Add /schema/details endpoint, which includes field cardinality computed via Count(Distinct()) --- api.go | 38 +++++++++++++++++++++++++++++++++++--- field.go | 9 +++++---- holder.go | 9 +++++---- http/handler.go | 15 +++++++++++++++ row.go | 4 ++++ 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/api.go b/api.go index d2c09fe85..025d37537 100644 --- a/api.go +++ b/api.go @@ -155,14 +155,20 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er return QueryResponse{}, errors.Wrap(err, "validating api method") } + if !req.Remote { + defer api.tracker.Finish(api.tracker.Start(req.Query, req.SQLQuery, api.server.nodeID, req.Index, start)) + } + + return api.query(ctx, req) +} + +// query provides query functionality for internal use, without tracing, validation, or tracking +func (api *API) query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() if err != nil { return QueryResponse{}, errors.Wrap(err, "parsing") } - if !req.Remote { - defer api.tracker.Finish(api.tracker.Start(req.Query, req.SQLQuery, api.server.nodeID, req.Index, start)) - } // TODO can we get rid of exec options and pass the QueryRequest directly to executor? execOpts := &execOptions{ Remote: req.Remote, @@ -989,6 +995,32 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.holder.limitedSchema() } +// SchemaDetails returns information about each index in Pilosa including which +// fields they contain, and additional field information such as cardinality +func (api *API) SchemaDetails(ctx context.Context) []*IndexInfo { + span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") + defer span.Finish() + schema := api.holder.limitedSchema() + for _, index := range schema { + for _, field := range index.Fields { + q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name) + req := QueryRequest{Index: index.Name, Query: q} + resp, err := api.query(ctx, &req) + if err != nil { + // TODO ? + continue + } + if len(resp.Results) == 0 { + continue + } + if card, ok := resp.Results[0].(uint64); ok { + field.Cardinality = card + } + } + } + return schema +} + // ApplySchema takes the given schema and applies it across the // cluster (if remote is false), or just to this node (if remote is // true). This is designed for the use case of replicating a schema diff --git a/field.go b/field.go index baec3a931..494057560 100644 --- a/field.go +++ b/field.go @@ -1867,10 +1867,11 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FieldInfo represents schema information for a field. type FieldInfo struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options FieldOptions `json:"options"` - Views []*ViewInfo `json:"views,omitempty"` + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options FieldOptions `json:"options"` + Cardinality uint64 `json:"cardinality,omitempty"` + Views []*ViewInfo `json:"views,omitempty"` } type fieldInfoSlice []*FieldInfo diff --git a/holder.go b/holder.go index 3e40ea14b..9dc14e6a3 100644 --- a/holder.go +++ b/holder.go @@ -851,9 +851,10 @@ func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{ - Name: index.Name(), - CreatedAt: index.CreatedAt(), - Options: index.Options(), + Name: index.Name(), + CreatedAt: index.CreatedAt(), + Options: index.Options(), + ShardWidth: ShardWidth, } for _, field := range index.Fields() { fi := &FieldInfo{ @@ -874,7 +875,7 @@ func (h *Holder) Schema() []*IndexInfo { return a } -// limitedSchema returns schema information for all indexes and fields. +// limitedSchema returns schema information for all non-hidden indexes and fields. func (h *Holder) limitedSchema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { diff --git a/http/handler.go b/http/handler.go index a6dfd5828..14ef35fe1 100644 --- a/http/handler.go +++ b/http/handler.go @@ -392,6 +392,7 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/inspect", handler.handleInspect).Methods("GET").Name("Inspect") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") + router.HandleFunc("/schema/details", handler.handleGetSchemaDetails).Methods("GET").Name("GetSchemaDetails") router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus") router.HandleFunc("/transaction", handler.handlePostTransaction).Methods("POST").Name("PostTransaction") @@ -672,6 +673,20 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { } } +// handleGetSchema handles GET /schema/details requests. +func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + + w.Header().Set("Content-Type", "application/json") + schema := h.api.SchemaDetails(r.Context()) + if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { + h.logger.Printf("write schema response error: %s", err) + } +} + func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() remoteStr := q.Get("remote") diff --git a/row.go b/row.go index 3ff47cba5..5e946d09a 100644 --- a/row.go +++ b/row.go @@ -461,6 +461,10 @@ func (r *Row) invalidateCount() { // Count returns the number of columns in the row. func (r *Row) Count() uint64 { var n uint64 + if r == nil { + // Count(Distinct()) on an empty field panics here + return n + } for i := range r.segments { n += r.segments[i].Count() } From 246c345c493f8d7466e20ed32f813c5575f779d4 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 4 Feb 2021 22:49:42 -0600 Subject: [PATCH 24/29] Eliminate omitempty from /schema/details response type --- api.go | 6 ++-- cluster.go | 9 +++-- field.go | 24 ++++++++++--- holder.go | 39 ++++++++++++++------- http/handler.go | 2 +- index.go | 15 ++++++++ server/handler_test.go | 78 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 149 insertions(+), 24 deletions(-) diff --git a/api.go b/api.go index 025d37537..6fa885e01 100644 --- a/api.go +++ b/api.go @@ -992,15 +992,15 @@ func (err MessageProcessingError) Unwrap() error { func (api *API) Schema(ctx context.Context) []*IndexInfo { span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") defer span.Finish() - return api.holder.limitedSchema() + return api.holder.Schema(false) } // SchemaDetails returns information about each index in Pilosa including which // fields they contain, and additional field information such as cardinality -func (api *API) SchemaDetails(ctx context.Context) []*IndexInfo { +func (api *API) SchemaDetails(ctx context.Context) []*IndexDetails { span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") defer span.Finish() - schema := api.holder.limitedSchema() + schema := api.holder.SchemaDetails() for _, index := range schema { for _, field := range index.Fields { q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name) diff --git a/cluster.go b/cluster.go index 3a9976e72..45b93ef67 100644 --- a/cluster.go +++ b/cluster.go @@ -625,7 +625,7 @@ func (c *cluster) unprotectedStatus() *ClusterStatus { ClusterID: c.id, State: c.state, Nodes: c.nodes, - Schema: &Schema{Indexes: c.holder.Schema()}, + Schema: &Schema{Indexes: c.holder.Schema(true)}, } } @@ -2372,7 +2372,7 @@ func (c *cluster) nodeLeave(nodeID string) error { func (c *cluster) nodeStatus() *NodeStatus { ns := &NodeStatus{ Node: c.Node, - Schema: &Schema{Indexes: c.holder.Schema()}, + Schema: &Schema{Indexes: c.holder.Schema(true)}, } var availableShards *roaring.Bitmap for _, idx := range ns.Schema.Indexes { @@ -3134,6 +3134,11 @@ type Schema struct { Indexes []*IndexInfo `json:"indexes"` } +// SchemaDetails contains information about indexes and their configuration. +type SchemaDetails struct { + Indexes []*IndexDetails `json:"indexes"` +} + func encodeTopology(topology *Topology) *internal.Topology { if topology == nil { return nil diff --git a/field.go b/field.go index 494057560..f4ab3982b 100644 --- a/field.go +++ b/field.go @@ -1867,11 +1867,10 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FieldInfo represents schema information for a field. type FieldInfo struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options FieldOptions `json:"options"` - Cardinality uint64 `json:"cardinality,omitempty"` - Views []*ViewInfo `json:"views,omitempty"` + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options FieldOptions `json:"options"` + Views []*ViewInfo `json:"views,omitempty"` } type fieldInfoSlice []*FieldInfo @@ -1880,6 +1879,21 @@ func (p fieldInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p fieldInfoSlice) Len() int { return len(p) } func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } +// FieldDetails represents detailed schema information for a field. +type FieldDetails struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options FieldOptions `json:"options"` + Cardinality uint64 `json:"cardinality"` + Views []*ViewInfo `json:"views,omitempty"` +} + +type fieldDetailsSlice []*FieldDetails + +func (p fieldDetailsSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p fieldDetailsSlice) Len() int { return len(p) } +func (p fieldDetailsSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } + // FieldOptions represents options to set when initializing a field. type FieldOptions struct { Base int64 `json:"base,omitempty"` diff --git a/holder.go b/holder.go index 9dc14e6a3..1bbb33984 100644 --- a/holder.go +++ b/holder.go @@ -847,7 +847,9 @@ func (h *Holder) availableShardsByIndex() map[string]*roaring.Bitmap { } // Schema returns schema information for all indexes, fields, and views. -func (h *Holder) Schema() []*IndexInfo { +// If includeHiddenAndViews=true, include fields beginning with "_", +// as well as view details. +func (h *Holder) Schema(includeHiddenAndViews bool) []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{ @@ -857,15 +859,20 @@ func (h *Holder) Schema() []*IndexInfo { ShardWidth: ShardWidth, } for _, field := range index.Fields() { + if !includeHiddenAndViews && strings.HasPrefix(field.name, "_") { + continue + } fi := &FieldInfo{ Name: field.Name(), CreatedAt: field.CreatedAt(), Options: field.Options(), } - for _, view := range field.views() { - fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) + if includeHiddenAndViews { + for _, view := range field.views() { + fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) + } + sort.Sort(viewInfoSlice(fi.Views)) } - sort.Sort(viewInfoSlice(fi.Views)) di.Fields = append(di.Fields, fi) } sort.Sort(fieldInfoSlice(di.Fields)) @@ -875,32 +882,38 @@ func (h *Holder) Schema() []*IndexInfo { return a } -// limitedSchema returns schema information for all non-hidden indexes and fields. -func (h *Holder) limitedSchema() []*IndexInfo { - var a []*IndexInfo +// SchemaDetails returns schema information for all non-hidden indexes and fields, +// including additional per-field details such as cardinality, actual range of integer data, etc. +// This function duplicates the logic of Holder.Schema because the FieldDetails struct +// includes a struct-field for cardinality, with default value 0, so the behavior of omitempty +// is incompatible between the /schema and /schema/details HTTP endpoints. A value of 0 for +// cardinality is meaningful, so it should be included when accurate, and not accidentally +// reported as 0 when the struct-field has not been populated. +func (h *Holder) SchemaDetails() []*IndexDetails { + var a []*IndexDetails for _, index := range h.Indexes() { - di := &IndexInfo{ + di := &IndexDetails{ Name: index.Name(), CreatedAt: index.CreatedAt(), Options: index.Options(), ShardWidth: ShardWidth, - Fields: make([]*FieldInfo, 0, len(index.Fields())), + Fields: make([]*FieldDetails, 0, len(index.Fields())), } for _, field := range index.Fields() { if strings.HasPrefix(field.name, "_") { continue } - fi := &FieldInfo{ + fi := &FieldDetails{ Name: field.Name(), CreatedAt: field.CreatedAt(), Options: field.Options(), } di.Fields = append(di.Fields, fi) } - sort.Sort(fieldInfoSlice(di.Fields)) + sort.Sort(fieldDetailsSlice(di.Fields)) a = append(a, di) } - sort.Sort(indexInfoSlice(a)) + sort.Sort(indexDetailsSlice(a)) return a } @@ -1337,7 +1350,7 @@ func (s *holderSyncer) SyncHolder() error { defer s.mu.Unlock() ti := time.Now() // Iterate over schema in sorted order. - for _, di := range s.Holder.Schema() { + for _, di := range s.Holder.Schema(true) { // Verify syncer has not closed. if s.IsClosing() { return nil diff --git a/http/handler.go b/http/handler.go index 14ef35fe1..2d98a5ec4 100644 --- a/http/handler.go +++ b/http/handler.go @@ -682,7 +682,7 @@ func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) w.Header().Set("Content-Type", "application/json") schema := h.api.SchemaDetails(r.Context()) - if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { + if err := json.NewEncoder(w).Encode(pilosa.SchemaDetails{Indexes: schema}); err != nil { h.logger.Printf("write schema response error: %s", err) } } diff --git a/index.go b/index.go index 6129289f5..a290e2d3d 100644 --- a/index.go +++ b/index.go @@ -668,6 +668,21 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p indexInfoSlice) Len() int { return len(p) } func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } +// IndexDetails represents detailed schema information for an index. +type IndexDetails struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options IndexOptions `json:"options"` + Fields []*FieldDetails `json:"fields"` + ShardWidth uint64 `json:"shardWidth"` +} + +type indexDetailsSlice []*IndexDetails + +func (p indexDetailsSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p indexDetailsSlice) Len() int { return len(p) } +func (p indexDetailsSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } + // IndexOptions represents options to set when initializing an index. type IndexOptions struct { Keys bool `json:"keys"` diff --git a/server/handler_test.go b/server/handler_test.go index 834399587..8c0c83530 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -112,6 +112,19 @@ func TestHandler_Endpoints(t *testing.T) { }) + t.Run("SchemaDetailsEmpty", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + body := w.Body.String() + if body != "{\"indexes\":null}\n" { + t.Fatalf("unexpected empty schema: '%v'", body) + } + + }) + t.Run("PostSchema", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`))) @@ -211,6 +224,57 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatal(err) } + // i2 is for SchemaDetails + i2 := hldr.MustCreateIndexIfNotExists("i2", pilosa.IndexOptions{}) + tx2, err := holder.BeginTx(true, i2.Index, shard) + if err != nil { + t.Fatal(err) + } + defer tx2.Rollback() + if f, err := i2.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000)); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + if f, err := i2.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeInt(-100, 100)); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + + if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i2", Query: "Set(10, f1=4) Set(11, f1=5) Set(12, f1=6) Set(13, f1=7)"}); err != nil { + t.Fatal(err) + } + + if f, err := i2.CreateFieldIfNotExists("f2", pilosa.OptFieldTypeDecimal(1, pql.Decimal{Value: -10}, pql.Decimal{Value: 10})); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i2", Query: "Set(10, f2=4) Set(11, f2=5) Set(12, f2=6) Set(13, f2=7) Set(14, f2=8)"}); err != nil { + t.Fatal(err) + } + + if f, err := i2.CreateFieldIfNotExists("f3", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + if f, err := i2.CreateFieldIfNotExists("f4", pilosa.OptFieldTypeMutex(pilosa.CacheTypeRanked, 5000)); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + if f, err := i2.CreateFieldIfNotExists("f5", pilosa.OptFieldTypeBool()); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + t.Fatal(err) + } + + if err := tx2.Commit(); err != nil { + t.Fatal(err) + } + t.Run("Schema", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) @@ -225,6 +289,20 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("SchemaDetails", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + body := strings.TrimSpace(w.Body.String()) + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":0},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":1048576},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":1048576},{"name":"i2","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":1000,"keys":false},"cardinality":1},{"name":"f1","options":{"type":"int","base":0,"bitDepth":3,"min":-100,"max":100,"keys":false,"foreignIndex":""},"cardinality":4},{"name":"f2","options":{"type":"decimal","base":0,"scale":1,"bitDepth":7,"min":-10,"max":10,"keys":false},"cardinality":5},{"name":"f3","options":{"type":"time","timeQuantum":"YMDH","keys":false,"noStandardView":false},"cardinality":1},{"name":"f4","options":{"type":"mutex","cacheType":"ranked","cacheSize":5000,"keys":false},"cardinality":1},{"name":"f5","options":{"type":"bool"},"cardinality":1}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) + if body != target { + t.Fatalf("%s\n!=\n%s", target, body) + } + }) + t.Run("Import", func(t *testing.T) { indexInfo := cmd.API.Schema(context.Background()) err := cmd.API.ApplySchema(context.Background(), &pilosa.Schema{Indexes: indexInfo}, false) From 12462886a629a468a144987c0689490bd42e4b7b Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 4 Feb 2021 22:50:07 -0600 Subject: [PATCH 25/29] Shorten shard lists in error messages --- executor.go | 89 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/executor.go b/executor.go index 6a50d4c2c..bb17f6261 100644 --- a/executor.go +++ b/executor.go @@ -643,6 +643,41 @@ func (e *executor) preprocessQuery(ctx context.Context, qcx *Qcx, index string, } } +type shardSlice []uint64 + +// String creates a run-length encoded representation of a slice of shard IDs (integers). +// For example, []uint64{0, 1, 3, 4, 5, 7, 8, 9, 11, 13} is represented as +// [0-1,3-5,7-9,11,13]. +func (s shardSlice) String() string { + if len(s) == 0 { + // surely this is impossible + return "[]" + } + runs := make([]string, 0, len(s)/2) + start := s[0] + end := start + for n := 1; n < len(s); n++ { + if s[n] == end+1 { + end = s[n] + } else { + repr := fmt.Sprintf("%d", start) + if end > start { + repr += fmt.Sprintf("-%d", end) + } + runs = append(runs, repr) + start = s[n] + end = start + } + } + repr := fmt.Sprintf("%d", start) + if end > start { + repr += fmt.Sprintf("-%d", end) + } + runs = append(runs, repr) + + return "[" + strings.Join(runs, ",") + "]" +} + // executeCall executes a call. func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall") @@ -692,47 +727,49 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p case "Sum": statFn() res, err := e.executeSum(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSum %v", shards) + return res, errors.Wrapf(err, "executeSum %v", shardSlice(shards)) case "Min": statFn() res, err := e.executeMin(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMin %v", shards) + return res, errors.Wrapf(err, "executeMin %v", shardSlice(shards)) case "Max": statFn() res, err := e.executeMax(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMax %v", shards) + return res, errors.Wrapf(err, "executeMax %v", shardSlice(shards)) case "MinRow": statFn() res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMinRow %v", shards) + return res, errors.Wrapf(err, "executeMinRow %v", shardSlice(shards)) case "MaxRow": statFn() res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeMaxRow %v", shards) + return res, errors.Wrapf(err, "executeMaxRow %v", shardSlice(shards)) case "Clear": statFn() res, err := e.executeClearBit(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeClearBit %v", shards) + return res, errors.Wrapf(err, "executeClearBit %v", shardSlice(shards)) case "ClearRow": statFn() res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeClearRow %v", shards) + return res, errors.Wrapf(err, "executeClearRow %v", shardSlice(shards)) case "Distinct": statFn() res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeDistinct %v", shards) + // TODO this can produce an ugly list of 256 shards + return res, errors.Wrapf(err, "executeDistinct %v", shardSlice(shards)) case "Store": statFn() res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeSetRow %v", shards) + return res, errors.Wrapf(err, "executeSetRow %v", shardSlice(shards)) case "Count": statFn() res, err := e.executeCount(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeCount %v", shards) + // TODO this can produce an ugly list of 256 shards + return res, errors.Wrapf(err, "executeCount %v", shardSlice(shards)) case "Set": statFn() res, err := e.executeSet(ctx, qcx, index, c, opt) - return res, errors.Wrapf(err, "executeSet %v", shards) + return res, errors.Wrapf(err, "executeSet %v", shardSlice(shards)) case "SetRowAttrs": statFn() return nil, errors.Wrap(e.executeSetRowAttrs(ctx, qcx, index, c, opt), "executeSetRowAttrs") @@ -742,50 +779,50 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p case "TopK": statFn() res, err := e.executeTopK(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopK %v", shards) + return res, errors.Wrapf(err, "executeTopK %v", shardSlice(shards)) case "TopN": statFn() res, err := e.executeTopN(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeTopN %v", shards) + return res, errors.Wrapf(err, "executeTopN %v", shardSlice(shards)) case "Rows": statFn() res, err := e.executeRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeRows %v", shards) + return res, errors.Wrapf(err, "executeRows %v", shardSlice(shards)) case "Extract": statFn() res, err := e.executeExtract(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeExtract %v", shards) + return res, errors.Wrapf(err, "executeExtract %v", shardSlice(shards)) case "GroupBy": statFn() res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeGroupBy %v", shards) + return res, errors.Wrapf(err, "executeGroupBy %v", shardSlice(shards)) case "Options": statFn() res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeOptionsCall %v", shards) + return res, errors.Wrapf(err, "executeOptionsCall %v", shardSlice(shards)) case "IncludesColumn": res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shards) + return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shardSlice(shards)) case "FieldValue": statFn() res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeFieldValueCall %v", shards) + return res, errors.Wrapf(err, "executeFieldValueCall %v", shardSlice(shards)) case "Precomputed": res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executePrecomputedCall %v", shards) + return res, errors.Wrapf(err, "executePrecomputedCall %v", shardSlice(shards)) case "UnionRows": res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeUnionRows %v", shards) + return res, errors.Wrapf(err, "executeUnionRows %v", shardSlice(shards)) case "ConstRow": res, err := e.executeConstRow(ctx, index, c) - return res, errors.Wrapf(err, "executeConstRow %v", shards) + return res, errors.Wrapf(err, "executeConstRow %v", shardSlice(shards)) case "Limit": res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeLimitCall %v", shards) + return res, errors.Wrapf(err, "executeLimitCall %v", shardSlice(shards)) default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap. statFn() res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt) - return res, errors.Wrapf(err, "executeBitmapCall %v", shards) + return res, errors.Wrapf(err, "executeBitmapCall %v", shardSlice(shards)) } } @@ -2856,7 +2893,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, errors.Wrapf(err, "mapReduce shards: %v", shards) + return nil, errors.Wrapf(err, "mapReduce shards: %v", shardSlice(shards)) } results, _ := other.([]GroupCount) @@ -5595,7 +5632,7 @@ func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch cha // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) if err != nil { - return errors.Wrapf(err, "shards by node %v", shards) + return errors.Wrapf(err, "shards by node %v", shardSlice(shards)) } // Execute each node in a separate goroutine. From 27739991908d298a4bb138beb824b91ff6df0d16 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 8 Feb 2021 12:12:34 -0600 Subject: [PATCH 26/29] Simplify response structs --- api.go | 6 +++--- cluster.go | 5 ----- executor.go | 2 -- field.go | 24 +++++------------------- holder.go | 35 ----------------------------------- http/handler.go | 2 +- index.go | 15 --------------- server/handler_test.go | 38 +++++++++++++++++++------------------- 8 files changed, 28 insertions(+), 99 deletions(-) diff --git a/api.go b/api.go index 6fa885e01..ff5b7c6b4 100644 --- a/api.go +++ b/api.go @@ -997,10 +997,10 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { // SchemaDetails returns information about each index in Pilosa including which // fields they contain, and additional field information such as cardinality -func (api *API) SchemaDetails(ctx context.Context) []*IndexDetails { +func (api *API) SchemaDetails(ctx context.Context) []*IndexInfo { span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") defer span.Finish() - schema := api.holder.SchemaDetails() + schema := api.holder.Schema(false) for _, index := range schema { for _, field := range index.Fields { q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name) @@ -1014,7 +1014,7 @@ func (api *API) SchemaDetails(ctx context.Context) []*IndexDetails { continue } if card, ok := resp.Results[0].(uint64); ok { - field.Cardinality = card + field.Cardinality = &card } } } diff --git a/cluster.go b/cluster.go index 45b93ef67..82e2e67a1 100644 --- a/cluster.go +++ b/cluster.go @@ -3134,11 +3134,6 @@ type Schema struct { Indexes []*IndexInfo `json:"indexes"` } -// SchemaDetails contains information about indexes and their configuration. -type SchemaDetails struct { - Indexes []*IndexDetails `json:"indexes"` -} - func encodeTopology(topology *Topology) *internal.Topology { if topology == nil { return nil diff --git a/executor.go b/executor.go index bb17f6261..34fc5c87a 100644 --- a/executor.go +++ b/executor.go @@ -755,7 +755,6 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p case "Distinct": statFn() res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt) - // TODO this can produce an ugly list of 256 shards return res, errors.Wrapf(err, "executeDistinct %v", shardSlice(shards)) case "Store": statFn() @@ -764,7 +763,6 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p case "Count": statFn() res, err := e.executeCount(ctx, qcx, index, c, shards, opt) - // TODO this can produce an ugly list of 256 shards return res, errors.Wrapf(err, "executeCount %v", shardSlice(shards)) case "Set": statFn() diff --git a/field.go b/field.go index f4ab3982b..e3000ca61 100644 --- a/field.go +++ b/field.go @@ -1867,10 +1867,11 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FieldInfo represents schema information for a field. type FieldInfo struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options FieldOptions `json:"options"` - Views []*ViewInfo `json:"views,omitempty"` + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options FieldOptions `json:"options"` + Cardinality *uint64 `json:"cardinality,omitempty"` + Views []*ViewInfo `json:"views,omitempty"` } type fieldInfoSlice []*FieldInfo @@ -1879,21 +1880,6 @@ func (p fieldInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p fieldInfoSlice) Len() int { return len(p) } func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } -// FieldDetails represents detailed schema information for a field. -type FieldDetails struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options FieldOptions `json:"options"` - Cardinality uint64 `json:"cardinality"` - Views []*ViewInfo `json:"views,omitempty"` -} - -type fieldDetailsSlice []*FieldDetails - -func (p fieldDetailsSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p fieldDetailsSlice) Len() int { return len(p) } -func (p fieldDetailsSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } - // FieldOptions represents options to set when initializing a field. type FieldOptions struct { Base int64 `json:"base,omitempty"` diff --git a/holder.go b/holder.go index 1bbb33984..46bacfa93 100644 --- a/holder.go +++ b/holder.go @@ -882,41 +882,6 @@ func (h *Holder) Schema(includeHiddenAndViews bool) []*IndexInfo { return a } -// SchemaDetails returns schema information for all non-hidden indexes and fields, -// including additional per-field details such as cardinality, actual range of integer data, etc. -// This function duplicates the logic of Holder.Schema because the FieldDetails struct -// includes a struct-field for cardinality, with default value 0, so the behavior of omitempty -// is incompatible between the /schema and /schema/details HTTP endpoints. A value of 0 for -// cardinality is meaningful, so it should be included when accurate, and not accidentally -// reported as 0 when the struct-field has not been populated. -func (h *Holder) SchemaDetails() []*IndexDetails { - var a []*IndexDetails - for _, index := range h.Indexes() { - di := &IndexDetails{ - Name: index.Name(), - CreatedAt: index.CreatedAt(), - Options: index.Options(), - ShardWidth: ShardWidth, - Fields: make([]*FieldDetails, 0, len(index.Fields())), - } - for _, field := range index.Fields() { - if strings.HasPrefix(field.name, "_") { - continue - } - fi := &FieldDetails{ - Name: field.Name(), - CreatedAt: field.CreatedAt(), - Options: field.Options(), - } - di.Fields = append(di.Fields, fi) - } - sort.Sort(fieldDetailsSlice(di.Fields)) - a = append(a, di) - } - sort.Sort(indexDetailsSlice(a)) - return a -} - // applySchema applies an internal Schema to Holder. func (h *Holder) applySchema(schema *Schema) error { // Create indexes that don't exist. diff --git a/http/handler.go b/http/handler.go index 2d98a5ec4..14ef35fe1 100644 --- a/http/handler.go +++ b/http/handler.go @@ -682,7 +682,7 @@ func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) w.Header().Set("Content-Type", "application/json") schema := h.api.SchemaDetails(r.Context()) - if err := json.NewEncoder(w).Encode(pilosa.SchemaDetails{Indexes: schema}); err != nil { + if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { h.logger.Printf("write schema response error: %s", err) } } diff --git a/index.go b/index.go index a290e2d3d..6129289f5 100644 --- a/index.go +++ b/index.go @@ -668,21 +668,6 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p indexInfoSlice) Len() int { return len(p) } func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } -// IndexDetails represents detailed schema information for an index. -type IndexDetails struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options IndexOptions `json:"options"` - Fields []*FieldDetails `json:"fields"` - ShardWidth uint64 `json:"shardWidth"` -} - -type indexDetailsSlice []*IndexDetails - -func (p indexDetailsSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p indexDetailsSlice) Len() int { return len(p) } -func (p indexDetailsSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } - // IndexOptions represents options to set when initializing an index. type IndexOptions struct { Keys bool `json:"keys"` diff --git a/server/handler_test.go b/server/handler_test.go index 8c0c83530..035932699 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -224,6 +224,20 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatal(err) } + t.Run("Schema", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + body := strings.TrimSpace(w.Body.String()) + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) + if body != target { + t.Fatalf("\n%s\n!=\n%s", target, body) + } + }) + // i2 is for SchemaDetails i2 := hldr.MustCreateIndexIfNotExists("i2", pilosa.IndexOptions{}) tx2, err := holder.BeginTx(true, i2.Index, shard) @@ -275,20 +289,6 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatal(err) } - t.Run("Schema", func(t *testing.T) { - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - body := strings.TrimSpace(w.Body.String()) - target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) - if body != target { - t.Fatalf("%s != %s", target, body) - } - }) - t.Run("SchemaDetails", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil)) @@ -297,7 +297,7 @@ func TestHandler_Endpoints(t *testing.T) { } body := strings.TrimSpace(w.Body.String()) - target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":0},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":1048576},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":1048576},{"name":"i2","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":1000,"keys":false},"cardinality":1},{"name":"f1","options":{"type":"int","base":0,"bitDepth":3,"min":-100,"max":100,"keys":false,"foreignIndex":""},"cardinality":4},{"name":"f2","options":{"type":"decimal","base":0,"scale":1,"bitDepth":7,"min":-10,"max":10,"keys":false},"cardinality":5},{"name":"f3","options":{"type":"time","timeQuantum":"YMDH","keys":false,"noStandardView":false},"cardinality":1},{"name":"f4","options":{"type":"mutex","cacheType":"ranked","cacheSize":5000,"keys":false},"cardinality":1},{"name":"f5","options":{"type":"bool"},"cardinality":1}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":0},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":%[1]d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":%[1]d},{"name":"i2","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":1000,"keys":false},"cardinality":1},{"name":"f1","options":{"type":"int","base":0,"bitDepth":3,"min":-100,"max":100,"keys":false,"foreignIndex":""},"cardinality":4},{"name":"f2","options":{"type":"decimal","base":0,"scale":1,"bitDepth":7,"min":-10,"max":10,"keys":false},"cardinality":5},{"name":"f3","options":{"type":"time","timeQuantum":"YMDH","keys":false,"noStandardView":false},"cardinality":1},{"name":"f4","options":{"type":"mutex","cacheType":"ranked","cacheSize":5000,"keys":false},"cardinality":1},{"name":"f5","options":{"type":"bool"},"cardinality":1}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) if body != target { t.Fatalf("%s\n!=\n%s", target, body) } @@ -476,13 +476,13 @@ func TestHandler_Endpoints(t *testing.T) { for _, nodeUsage := range nodeUsages { numIndexes := len(nodeUsage.Disk.IndexUsage) - if nodeUsage.Disk.TotalUse < 75000 || nodeUsage.Disk.TotalUse > 300000 { + if nodeUsage.Disk.TotalUse < 75000 || nodeUsage.Disk.TotalUse > 500000 { // Usage measurements are not consistent between machines, or // over time, as features and implementations change, so checking // for a range of sizes may be most useful way to test the details of this. - t.Fatalf("expected 75k < total < 300k, got %d", nodeUsage.Disk.TotalUse) + t.Fatalf("expected 75k < total < 500k, got %d", nodeUsage.Disk.TotalUse) } - if numIndexes != 2 { + if numIndexes != 3 { t.Fatalf("wrong length index usage list: expected %d, got %d", 2, numIndexes) } numFields := len(nodeUsage.Disk.IndexUsage["i1"].Fields) @@ -561,7 +561,7 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/internal/shards/max", nil)) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0,"i2":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } }) From dc6c92771b133478272e63e5049375bff0f85cbf Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 8 Feb 2021 17:25:38 -0600 Subject: [PATCH 27/29] Fail entire /schema/details request if one field query fails --- api.go | 7 +++---- http/handler.go | 6 +++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/api.go b/api.go index ff5b7c6b4..55ea78af4 100644 --- a/api.go +++ b/api.go @@ -997,7 +997,7 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { // SchemaDetails returns information about each index in Pilosa including which // fields they contain, and additional field information such as cardinality -func (api *API) SchemaDetails(ctx context.Context) []*IndexInfo { +func (api *API) SchemaDetails(ctx context.Context) ([]*IndexInfo, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") defer span.Finish() schema := api.holder.Schema(false) @@ -1007,8 +1007,7 @@ func (api *API) SchemaDetails(ctx context.Context) []*IndexInfo { req := QueryRequest{Index: index.Name, Query: q} resp, err := api.query(ctx, &req) if err != nil { - // TODO ? - continue + return schema, errors.Wrapf(err, "querying cardinality (%s/%s)", index.Name, field.Name) } if len(resp.Results) == 0 { continue @@ -1018,7 +1017,7 @@ func (api *API) SchemaDetails(ctx context.Context) []*IndexInfo { } } } - return schema + return schema, nil } // ApplySchema takes the given schema and applies it across the diff --git a/http/handler.go b/http/handler.go index 14ef35fe1..5e8aad1ef 100644 --- a/http/handler.go +++ b/http/handler.go @@ -681,7 +681,11 @@ func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) } w.Header().Set("Content-Type", "application/json") - schema := h.api.SchemaDetails(r.Context()) + schema, err := h.api.SchemaDetails(r.Context()) + if err != nil { + h.logger.Printf("error getting detailed schema: %s", err) + return + } if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { h.logger.Printf("write schema response error: %s", err) } From 882b14444b12adc96ef0c9c223f03debbbd9d3e2 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 11 Feb 2021 14:37:04 -0600 Subject: [PATCH 28/29] Use consistent behavior in test to add queries to transaction --- server/handler_test.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/server/handler_test.go b/server/handler_test.go index 035932699..2ac74b71a 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -250,23 +250,27 @@ func TestHandler_Endpoints(t *testing.T) { } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { t.Fatal(err) } - if f, err := i2.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeInt(-100, 100)); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + + f, err := i2.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeInt(-100, 100)) + if err != nil { t.Fatal(err) } - if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i2", Query: "Set(10, f1=4) Set(11, f1=5) Set(12, f1=6) Set(13, f1=7)"}); err != nil { + for n := 0; n < 4; n++ { + if _, err := f.SetValue(tx2, uint64(n), int64(n)); err != nil { + t.Fatal(err) + } + } + + f, err = i2.CreateFieldIfNotExists("f2", pilosa.OptFieldTypeDecimal(1, pql.Decimal{Value: -10}, pql.Decimal{Value: 10})) + if err != nil { t.Fatal(err) } - if f, err := i2.CreateFieldIfNotExists("f2", pilosa.OptFieldTypeDecimal(1, pql.Decimal{Value: -10}, pql.Decimal{Value: 10})); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { - t.Fatal(err) - } - if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i2", Query: "Set(10, f2=4) Set(11, f2=5) Set(12, f2=6) Set(13, f2=7) Set(14, f2=8)"}); err != nil { - t.Fatal(err) + for n := 0; n < 5; n++ { + if _, err := f.SetValue(tx2, uint64(n), int64(n)); err != nil { + t.Fatal(err) + } } if f, err := i2.CreateFieldIfNotExists("f3", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil { @@ -297,9 +301,9 @@ func TestHandler_Endpoints(t *testing.T) { } body := strings.TrimSpace(w.Body.String()) - target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":0},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":%[1]d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":%[1]d},{"name":"i2","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":1000,"keys":false},"cardinality":1},{"name":"f1","options":{"type":"int","base":0,"bitDepth":3,"min":-100,"max":100,"keys":false,"foreignIndex":""},"cardinality":4},{"name":"f2","options":{"type":"decimal","base":0,"scale":1,"bitDepth":7,"min":-10,"max":10,"keys":false},"cardinality":5},{"name":"f3","options":{"type":"time","timeQuantum":"YMDH","keys":false,"noStandardView":false},"cardinality":1},{"name":"f4","options":{"type":"mutex","cacheType":"ranked","cacheSize":5000,"keys":false},"cardinality":1},{"name":"f5","options":{"type":"bool"},"cardinality":1}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":0},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":%[1]d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1}],"shardWidth":%[1]d},{"name":"i2","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":1000,"keys":false},"cardinality":1},{"name":"f1","options":{"type":"int","base":0,"bitDepth":2,"min":-100,"max":100,"keys":false,"foreignIndex":""},"cardinality":4},{"name":"f2","options":{"type":"decimal","base":0,"scale":1,"bitDepth":3,"min":-10,"max":10,"keys":false},"cardinality":5},{"name":"f3","options":{"type":"time","timeQuantum":"YMDH","keys":false,"noStandardView":false},"cardinality":1},{"name":"f4","options":{"type":"mutex","cacheType":"ranked","cacheSize":5000,"keys":false},"cardinality":1},{"name":"f5","options":{"type":"bool"},"cardinality":1}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) if body != target { - t.Fatalf("%s\n!=\n%s", target, body) + t.Fatalf("\n%s\n!=\n%s", target, body) } }) From 0e3461e9f447a005ab1f49bab8412d589c4ab513 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 12 Feb 2021 14:50:10 -0600 Subject: [PATCH 29/29] update lattice version to correct from accidental downgrade --- lattice | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lattice b/lattice index 28c2313ec..fa773628a 160000 --- a/lattice +++ b/lattice @@ -1 +1 @@ -Subproject commit 28c2313ecfcd7e083d42d4e409483e968b4c421b +Subproject commit fa773628a276e2590785a87fbc236c7e88ea6284