diff --git a/.travis.yml b/.travis.yml index 0e4f44d1a..673cf0a7f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: go go: - - 1.9 - "1.10" # Use string, as 1.10==1.1 if interpreted as float. - master env: @@ -31,12 +30,6 @@ matrix: fast_finish: true allow_failures: - go: master - - go: 1.9 - exclude: - - go: 1.9 - env: GOARCH=386 - - go: 1.9 - env: GOARCH=amd64 notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" diff --git a/CHANGELOG.md b/CHANGELOG.md index e95e6013e..6c044dcc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [v1.0.0] - 2018-07-06 +## [v1.0.0] - 2018-07-09 + +This version contains 218 contributions from 7 contributors. There are 184 files changed, 21,769 insertions, and 20,275 deletions. ### Added @@ -442,3 +444,6 @@ This version contains 53 contributions from 13 contributors (including 4 volunte [0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6 [0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7 [0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8 +[0.9.0]: https://github.com/pilosa/pilosa/compare/v0.8...v0.9 +[0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10 +[1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0 diff --git a/Dockerfile b/Dockerfile index 2b83c71c1..12e25ce45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.10.2 as builder +FROM golang:1.10.3 as builder COPY . /go/src/github.com/pilosa/pilosa/ diff --git a/Gopkg.lock b/Gopkg.lock index 33187bfe2..7c5513ee6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -11,19 +11,19 @@ branch = "master" name = "github.com/DataDog/datadog-go" packages = ["statsd"] - revision = "4d2e5696ebe914940bd7459d2266fb7d555ea1b7" + revision = "ef3a9daf849df2d7ee3bbf13808dfb481069a773" [[projects]] - branch = "master" name = "github.com/StackExchange/wmi" packages = ["."] - revision = "ea383cf3ba6ec950874b8486cd72356d007c768f" + revision = "5d049714c4a64225c3c79a7cf7d02f7fb5b96338" + version = "1.0.0" [[projects]] branch = "master" name = "github.com/armon/go-metrics" packages = ["."] - revision = "9a4b6e10bed6220a1665955aa2b75afc91eb10b3" + revision = "58588f401c2cc130a7308a52ca3bc6c0a76db04b" [[projects]] name = "github.com/boltdb/bolt" @@ -46,8 +46,8 @@ [[projects]] name = "github.com/fsnotify/fsnotify" packages = ["."] - revision = "629574ca2a5df945712d3079857300b5e4da0236" - version = "v1.4.2" + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" [[projects]] name = "github.com/go-ole/go-ole" @@ -55,20 +55,20 @@ ".", "oleutil" ] - revision = "0e87ea779d9deb219633b828a023b32e1244dd57" - version = "v1.2.0" + revision = "a41e3c4b706f6ae8dfbff342b06e40fa4d2d0506" + version = "v1.2.1" [[projects]] name = "github.com/gogo/protobuf" packages = ["proto"] - revision = "100ba4e885062801d56799d78530b73b178a78f3" - version = "v0.4" + revision = "1adfc126b41513cc696b209667c8656ea7aac67c" + version = "v1.0.0" [[projects]] - branch = "master" name = "github.com/golang/protobuf" packages = ["proto"] - revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9" + revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" + version = "v1.1.0" [[projects]] name = "github.com/google/go-cmp" @@ -85,8 +85,8 @@ [[projects]] name = "github.com/gorilla/context" packages = ["."] - revision = "1ea25387ff6f684839d82767c1733ff4d4d15d0a" - version = "v1.1" + revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" + version = "v1.1.1" [[projects]] name = "github.com/gorilla/handlers" @@ -97,8 +97,8 @@ [[projects]] name = "github.com/gorilla/mux" packages = ["."] - revision = "53c1911da2b537f792e7cafcb446b05ffe33b996" - version = "v1.6.1" + revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" + version = "v1.6.2" [[projects]] branch = "master" @@ -110,7 +110,7 @@ branch = "master" name = "github.com/hashicorp/go-immutable-radix" packages = ["."] - revision = "8aac2701530899b64bdea735a1de8da899815220" + revision = "7f3cd4390caab3250a57f30efdb2a65dd7649ecf" [[projects]] branch = "master" @@ -122,19 +122,19 @@ branch = "master" name = "github.com/hashicorp/go-multierror" packages = ["."] - revision = "83588e72410abfbe4df460eeb6f30841ae47d4c4" + revision = "b7773ae218740a7be65057fc60b366a49b538a44" [[projects]] branch = "master" name = "github.com/hashicorp/go-sockaddr" packages = ["."] - revision = "9b4c5fa5b10a683339a270d664474b9f4aee62fc" + revision = "6d291a969b86c4b633730bfc6b8b9d64c3aafed9" [[projects]] branch = "master" name = "github.com/hashicorp/golang-lru" packages = ["simplelru"] - revision = "0a025b7e63adc15a622f29b0b2c4c3848243bbf6" + revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3" [[projects]] branch = "master" @@ -143,6 +143,7 @@ ".", "hcl/ast", "hcl/parser", + "hcl/printer", "hcl/scanner", "hcl/strconv", "hcl/token", @@ -150,7 +151,7 @@ "json/scanner", "json/token" ] - revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" + revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" [[projects]] name = "github.com/hashicorp/memberlist" @@ -167,29 +168,26 @@ [[projects]] name = "github.com/magiconair/properties" packages = ["."] - revision = "be5ece7dd465ab0765a9682137865547526d1dfb" - version = "v1.7.3" + revision = "c2353362d570a7bfa228149c62842019201cfb71" + version = "v1.8.0" [[projects]] - branch = "master" name = "github.com/miekg/dns" - packages = [ - ".", - "internal/socket" - ] - revision = "9fc4eb252eedf0ef8adc05169ce35da5e31beaba" + packages = ["."] + revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1" + version = "v1.0.8" [[projects]] branch = "master" name = "github.com/mitchellh/mapstructure" packages = ["."] - revision = "06020f85339e21b2478f756a78e295255ffa4d6a" + revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b" [[projects]] name = "github.com/pelletier/go-toml" packages = ["."] - revision = "16398bac157da96aa88f98a2df640c7f32af1da2" - version = "v1.0.1" + revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" + version = "v1.2.0" [[projects]] name = "github.com/pkg/errors" @@ -200,8 +198,8 @@ [[projects]] name = "github.com/satori/go.uuid" packages = ["."] - revision = "879c5887cd475cd7864858769793b2ceb0d44feb" - version = "v1.1.0" + revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3" + version = "v1.2.0" [[projects]] branch = "master" @@ -219,8 +217,8 @@ "net", "process" ] - revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" - version = "v2.17.11" + revision = "4a180b209f5f494e5923cfce81ea30ba23915877" + version = "v2.18.06" [[projects]] branch = "master" @@ -229,55 +227,71 @@ revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" [[projects]] - branch = "master" name = "github.com/spf13/afero" packages = [ ".", "mem" ] - revision = "5660eeed305fe5f69c8fc6cf899132a459a97064" + revision = "787d034dfe70e44075ccc060d346146ef53270ad" + version = "v1.1.1" [[projects]] name = "github.com/spf13/cast" packages = ["."] - revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4" - version = "v1.1.0" + revision = "8965335b8c7107321228e3e3702cab9832751bac" + version = "v1.2.0" [[projects]] name = "github.com/spf13/cobra" packages = ["."] - revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" - version = "v0.0.1" + revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" + version = "v0.0.3" [[projects]] branch = "master" name = "github.com/spf13/jwalterweatherman" packages = ["."] - revision = "12bd96e66386c1960ab0f74ced1362f66f552f7b" + revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" [[projects]] name = "github.com/spf13/pflag" packages = ["."] - revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" - version = "v1.0.0" + revision = "583c0c0531f06d5278b7d917446061adc344b5cd" + version = "v1.0.1" [[projects]] name = "github.com/spf13/viper" packages = ["."] - revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" - version = "v1.0.0" + revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736" + version = "v1.0.2" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = [ + "ed25519", + "ed25519/internal/edwards25519" + ] + revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" [[projects]] branch = "master" name = "golang.org/x/net" - packages = ["context"] - revision = "a337091b0525af65de94df2eb7e98bd9962dcbe2" + packages = [ + "bpf", + "context", + "internal/iana", + "internal/socket", + "ipv4", + "ipv6" + ] + revision = "6f138e0f60713a248abf7046f1014a3ba90f5341" [[projects]] branch = "master" name = "golang.org/x/sync" packages = ["errgroup"] - revision = "fd80eb99c8f653c847d294a001bdf2a3a6f768f5" + revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" [[projects]] branch = "master" @@ -286,10 +300,9 @@ "unix", "windows" ] - revision = "1e2299c37cc91a509f1b12369872d27be0ce98a6" + revision = "1b2967e3c290b7c545b3db0deeda16e9be4f98a2" [[projects]] - branch = "master" name = "golang.org/x/text" packages = [ "internal/gen", @@ -299,13 +312,14 @@ "unicode/cldr", "unicode/norm" ] - revision = "88f656faf3f37f690df1a32515b479415e1a6769" + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" [[projects]] - branch = "v2" name = "gopkg.in/yaml.v2" packages = ["."] - revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f" + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" [solve-meta] analyzer-name = "dep" diff --git a/docs/installation.md b/docs/installation.md index 639293338..dd4d89780 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -42,7 +42,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v0.10.0-64-gf053d9a5 + Version: v1.0.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -71,19 +71,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.10.0/pilosa-v0.10.0-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.0.0/pilosa-v1.0.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.10.0-darwin-amd64.tar.gz + tar xfz pilosa-v1.0.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.10.0-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v1.0.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -100,7 +100,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v0.10.0-64-gf053d9a5 + Version: v1.0.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -163,7 +163,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v0.10.0-64-gf053d9a5 + Version: v1.0.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -222,19 +222,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.10.0/pilosa-v0.10.0-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.0.0/pilosa-v1.0.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.10.0-linux-amd64.tar.gz + tar xfz pilosa-v1.0.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.10.0-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v1.0.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -251,7 +251,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v0.10.0-64-gf053d9a5 + Version: v1.0.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -314,7 +314,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v0.10.0-64-gf053d9a5 + Version: v1.0.0 Build Time: 2018-05-14T22:14:01+0000 Usage: diff --git a/gossip/gossip.go b/gossip/gossip.go index 7650cff86..a5b4e9299 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -33,28 +33,28 @@ import ( ) // Ensure GossipMemberSet implements interfaces. -var _ memberlist.Delegate = &gossipMemberSet{} +var _ memberlist.Delegate = &memberSet{} -// gossipMemberSet represents a gossip implementation of MemberSet using memberlist. -type gossipMemberSet struct { +// memberSet represents a gossip implementation of MemberSet using memberlist. +type memberSet struct { mu sync.RWMutex memberlist *memberlist.Memberlist broadcasts *memberlist.TransmitLimitedQueue papi *pilosa.API - config *gossipConfig + config *config Logger pilosa.Logger logger *log.Logger transport *Transport - gossipEventReceiver *gossipEventReceiver + eventReceiver *eventReceiver } // Open implements the MemberSet interface to start network activity. -func (g *gossipMemberSet) Open() (err error) { +func (g *memberSet) Open() (err error) { g.mu.Lock() g.memberlist, err = memberlist.Create(g.config.memberlistConfig) g.mu.Unlock() @@ -93,8 +93,19 @@ func (g *gossipMemberSet) Open() (err error) { return nil } +// Close attempts to gracefully leave the cluster, and finally calls shutdown +// after (at most) a timeout period. +func (g *memberSet) Close() error { + leaveErr := g.memberlist.Leave(5 * time.Second) + shutdownErr := g.memberlist.Shutdown() + if leaveErr != nil || shutdownErr != nil { + return fmt.Errorf("leaving: '%v', shutting down: '%v'", leaveErr, shutdownErr) + } + return nil +} + // joinWithRetry wraps the standard memberlist Join function in a retry. -func (g *gossipMemberSet) joinWithRetry(hosts []string) error { +func (g *memberSet) joinWithRetry(hosts []string) error { err := retry(60, 2*time.Second, func() error { _, err := g.memberlist.Join(hosts) return err @@ -120,34 +131,34 @@ func retry(attempts int, sleep time.Duration, fn func() error) (err error) { //////////////////////////////////////////////////////////////// -type gossipConfig struct { +type config struct { gossipSeeds []string memberlistConfig *memberlist.Config } -// gossipMemberSetOption describes a functional option for GossipMemberSet. -type gossipMemberSetOption func(*gossipMemberSet) error +// memberSetOption describes a functional option for GossipMemberSet. +type memberSetOption func(*memberSet) error -// WithTransport is a functional option for providing a transport to NewGossipMemberSet. -func WithTransport(transport *Transport) gossipMemberSetOption { - return func(g *gossipMemberSet) error { +// WithTransport is a functional option for providing a transport to NewMemberSet. +func WithTransport(transport *Transport) memberSetOption { + return func(g *memberSet) error { g.transport = transport return nil } } -// WithLogger is a functional option for providing a logger to NewGossipMemberSet. -func WithLogger(logger *log.Logger) gossipMemberSetOption { - return func(g *gossipMemberSet) error { +// WithLogger is a functional option for providing a logger to NewMemberSet. +func WithLogger(logger *log.Logger) memberSetOption { + return func(g *memberSet) error { g.logger = logger return nil } } -// NewGossipMemberSet returns a new instance of GossipMemberSet based on options. -func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...gossipMemberSetOption) (*gossipMemberSet, error) { +// NewMemberSet returns a new instance of GossipMemberSet based on options. +func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*memberSet, error) { host := api.Node().URI.Host - g := &gossipMemberSet{ + g := &memberSet{ papi: api, Logger: pilosa.NopLogger, } @@ -158,8 +169,8 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...gossipMemberSetO return nil, errors.Wrap(err, "executing option") } } - ger := newGossipEventReceiver(g.logger, api) - g.gossipEventReceiver = ger + ger := newEventReceiver(g.logger, api) + g.eventReceiver = ger if g.transport == nil { port, err := strconv.Atoi(cfg.Port) @@ -210,7 +221,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...gossipMemberSetO conf.Events = ger conf.Logger = g.logger - g.config = &gossipConfig{ + g.config = &config{ memberlistConfig: conf, gossipSeeds: cfg.Seeds, } @@ -219,7 +230,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...gossipMemberSetO } // NodeMeta implementation of the memberlist.Delegate interface. -func (g *gossipMemberSet) NodeMeta(limit int) []byte { +func (g *memberSet) NodeMeta(limit int) []byte { buf, err := g.papi.Serializer.Marshal(g.papi.Node()) if err != nil { g.Logger.Printf("marshal message error: %s", err) @@ -230,7 +241,7 @@ func (g *gossipMemberSet) NodeMeta(limit int) []byte { // NotifyMsg implementation of the memberlist.Delegate interface // called when a user-data message is received. -func (g *gossipMemberSet) NotifyMsg(b []byte) { +func (g *memberSet) NotifyMsg(b []byte) { err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(b)) if err != nil { g.Logger.Printf("cluster message error: %s", err) @@ -239,13 +250,13 @@ func (g *gossipMemberSet) NotifyMsg(b []byte) { // GetBroadcasts implementation of the memberlist.Delegate interface // called when user data messages can be broadcast. -func (g *gossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { +func (g *memberSet) GetBroadcasts(overhead, limit int) [][]byte { return g.broadcasts.GetBroadcasts(overhead, limit) } // LocalState implementation of the memberlist.Delegate interface // sends this Node's state data. -func (g *gossipMemberSet) LocalState(join bool) []byte { +func (g *memberSet) LocalState(join bool) []byte { m := &pilosa.NodeStatus{ Node: g.papi.Node(), MaxShards: g.papi.MaxShards(context.Background()), @@ -263,28 +274,28 @@ func (g *gossipMemberSet) LocalState(join bool) []byte { // MergeRemoteState implementation of the memberlist.Delegate interface // receive and process the remote side's LocalState. -func (g *gossipMemberSet) MergeRemoteState(buf []byte, join bool) { +func (g *memberSet) MergeRemoteState(buf []byte, join bool) { err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)) if err != nil { g.Logger.Printf("merge state error: %s", err) } } -// gossipEventReceiver is used to enable an application to receive +// eventReceiver is used to enable an application to receive // events about joins and leaves over a channel. // // Care must be taken that events are processed in a timely manner from // the channel, since this delegate will block until an event can be sent. -type gossipEventReceiver struct { +type eventReceiver struct { ch chan memberlist.NodeEvent papi *pilosa.API logger *log.Logger } -// newGossipEventReceiver returns a new instance of GossipEventReceiver. -func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver { - ger := &gossipEventReceiver{ +// newEventReceiver returns a new instance of GossipEventReceiver. +func newEventReceiver(logger *log.Logger, papi *pilosa.API) *eventReceiver { + ger := &eventReceiver{ ch: make(chan memberlist.NodeEvent, 1), logger: logger, papi: papi, @@ -293,19 +304,19 @@ func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventRe return ger } -func (g *gossipEventReceiver) NotifyJoin(n *memberlist.Node) { +func (g *eventReceiver) NotifyJoin(n *memberlist.Node) { g.ch <- memberlist.NodeEvent{memberlist.NodeJoin, n} } -func (g *gossipEventReceiver) NotifyLeave(n *memberlist.Node) { +func (g *eventReceiver) NotifyLeave(n *memberlist.Node) { g.ch <- memberlist.NodeEvent{memberlist.NodeLeave, n} } -func (g *gossipEventReceiver) NotifyUpdate(n *memberlist.Node) { +func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) { g.ch <- memberlist.NodeEvent{memberlist.NodeUpdate, n} } -func (g *gossipEventReceiver) listen() { +func (g *eventReceiver) listen() { var nodeEventType pilosa.NodeEventType for { e := <-g.ch diff --git a/http/handler.go b/http/handler.go index 4946f69e0..d85d3a8a5 100644 --- a/http/handler.go +++ b/http/handler.go @@ -55,6 +55,8 @@ type Handler struct { ln net.Listener + closeTimeout time.Duration + server *http.Server } @@ -109,10 +111,20 @@ func OptHandlerListener(ln net.Listener) handlerOption { } } +// OptHandlerCloseTimeout controls how long to wait for the http Server to +// shutdown cleanly before forcibly destroying it. Default is 30 seconds. +func OptHandlerCloseTimeout(d time.Duration) handlerOption { + return func(h *Handler) error { + h.closeTimeout = d + return nil + } +} + // NewHandler returns a new instance of Handler with a default logger. func NewHandler(opts ...handlerOption) (*Handler, error) { handler := &Handler{ - logger: pilosa.NopLogger, + logger: pilosa.NopLogger, + closeTimeout: time.Second * 30, } handler.Handler = newRouter(handler) handler.populateValidators() @@ -146,10 +158,16 @@ func (h *Handler) Serve() error { return nil } +// Close tries to cleanly shutdown the HTTP server, and failing that, after a +// timeout, calls Server.Close. func (h *Handler) Close() error { - // TODO: timeout? - err := h.server.Shutdown(context.Background()) - return errors.Wrap(err, "shutdown http server") + deadlineCtx, cancelFunc := context.WithDeadline(context.Background(), time.Now().Add(h.closeTimeout)) + defer cancelFunc() + err := h.server.Shutdown(deadlineCtx) + if err != nil { + err = h.server.Close() + } + return errors.Wrap(err, "shutdown/close http server") } func (h *Handler) populateValidators() { diff --git a/internal/private.pb.go b/internal/private.pb.go index 8fdcfa7db..b6c7ff5ca 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: private.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -2141,24 +2140,6 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Private(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -3567,51 +3548,14 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var keykey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - keykey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey if m.Standard == nil { m.Standard = make(map[string]uint64) } - if iNdEx < postIndex { - var valuekey uint64 + var mapkey string + var mapvalue uint64 + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3621,31 +3565,69 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - valuekey |= (uint64(b) & 0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - var mapvalue uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy } - m.Standard[mapkey] = mapvalue - } else { - var mapvalue uint64 - m.Standard[mapkey] = mapvalue } + m.Standard[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex diff --git a/internal/public.pb.go b/internal/public.pb.go index cd1927f06..cceea4806 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: public.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -28,6 +27,8 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import binary "encoding/binary" + import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -799,7 +800,8 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue)))) + binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + i += 8 } return i, nil } @@ -1235,24 +1237,6 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Public(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Public(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2333,15 +2317,8 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } + v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 - v = uint64(dAtA[iNdEx-8]) - v |= uint64(dAtA[iNdEx-7]) << 8 - v |= uint64(dAtA[iNdEx-6]) << 16 - v |= uint64(dAtA[iNdEx-5]) << 24 - v |= uint64(dAtA[iNdEx-4]) << 32 - v |= uint64(dAtA[iNdEx-3]) << 40 - v |= uint64(dAtA[iNdEx-2]) << 48 - v |= uint64(dAtA[iNdEx-1]) << 56 m.FloatValue = float64(math.Float64frombits(v)) default: iNdEx = preIndex diff --git a/pql/pql.peg b/pql/pql.peg index 141d8f265..f55267e1f 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -57,6 +57,7 @@ posfield <- { p.addPosStr("_field", buffer[begin:end]) } uint <- [1-9] [0-9]* / '0' uintrow <- {p.addPosNum("_row", buffer[begin:end])} col <- ( {p.addPosNum("_col", buffer[begin:end])} + / '\'' '\'' {p.addPosStr("_col", buffer[begin:end])} / '"' '"' {p.addPosStr("_col", buffer[begin:end])} ) diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 2516def6c..7092b8f1f 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -94,6 +94,7 @@ const ( ruleAction41 ruleAction42 ruleAction43 + ruleAction44 ) var rul3s = [...]string{ @@ -176,6 +177,7 @@ var rul3s = [...]string{ "Action41", "Action42", "Action43", + "Action44", } type token32 struct { @@ -292,7 +294,7 @@ type PQL struct { Buffer string buffer []rune - rules [79]func() bool + rules [80]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -471,6 +473,8 @@ func (p *PQL) Execute() { case ruleAction42: p.addPosStr("_col", buffer[begin:end]) case ruleAction43: + p.addPosStr("_col", buffer[begin:end]) + case ruleAction44: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -632,7 +636,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction43, position) + add(ruleAction44, position) } add(ruletimestamp, position12) } @@ -1912,95 +1916,8 @@ func (p *PQL) Init() { position++ { position183 := position - { - position184 := position - l185: - { - position186, tokenIndex186 := position, tokenIndex - { - position187, tokenIndex187 := position, tokenIndex - { - position189, tokenIndex189 := position, tokenIndex - { - position190, tokenIndex190 := position, tokenIndex - if buffer[position] != rune('\'') { - goto l191 - } - position++ - goto l190 - l191: - position, tokenIndex = position190, tokenIndex190 - if buffer[position] != rune('\\') { - goto l192 - } - position++ - goto l190 - l192: - position, tokenIndex = position190, tokenIndex190 - if buffer[position] != rune('\n') { - goto l189 - } - position++ - } - l190: - goto l188 - l189: - position, tokenIndex = position189, tokenIndex189 - } - if !matchDot() { - goto l188 - } - goto l187 - l188: - position, tokenIndex = position187, tokenIndex187 - if buffer[position] != rune('\\') { - goto l193 - } - position++ - if buffer[position] != rune('n') { - goto l193 - } - position++ - goto l187 - l193: - position, tokenIndex = position187, tokenIndex187 - if buffer[position] != rune('\\') { - goto l194 - } - position++ - if buffer[position] != rune('"') { - goto l194 - } - position++ - goto l187 - l194: - position, tokenIndex = position187, tokenIndex187 - if buffer[position] != rune('\\') { - goto l195 - } - position++ - if buffer[position] != rune('\'') { - goto l195 - } - position++ - goto l187 - l195: - position, tokenIndex = position187, tokenIndex187 - if buffer[position] != rune('\\') { - goto l186 - } - position++ - if buffer[position] != rune('\\') { - goto l186 - } - position++ - } - l187: - goto l185 - l186: - position, tokenIndex = position186, tokenIndex186 - } - add(rulesinglequotedstring, position184) + if !_rules[rulesinglequotedstring]() { + goto l127 } add(rulePegText, position183) } @@ -2023,99 +1940,191 @@ func (p *PQL) Init() { /* 14 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ func() bool { { - position198 := position - l199: + position186 := position + l187: { - position200, tokenIndex200 := position, tokenIndex + position188, tokenIndex188 := position, tokenIndex { - position201, tokenIndex201 := position, tokenIndex + position189, tokenIndex189 := position, tokenIndex { - position203, tokenIndex203 := position, tokenIndex + position191, tokenIndex191 := position, tokenIndex { - position204, tokenIndex204 := position, tokenIndex + position192, tokenIndex192 := position, tokenIndex if buffer[position] != rune('"') { - goto l205 + goto l193 } position++ - goto l204 - l205: - position, tokenIndex = position204, tokenIndex204 + goto l192 + l193: + position, tokenIndex = position192, tokenIndex192 if buffer[position] != rune('\\') { - goto l206 + goto l194 } position++ - goto l204 - l206: - position, tokenIndex = position204, tokenIndex204 + goto l192 + l194: + position, tokenIndex = position192, tokenIndex192 if buffer[position] != rune('\n') { - goto l203 + goto l191 } position++ } - l204: - goto l202 - l203: - position, tokenIndex = position203, tokenIndex203 + l192: + goto l190 + l191: + position, tokenIndex = position191, tokenIndex191 } if !matchDot() { - goto l202 + goto l190 } - goto l201 - l202: - position, tokenIndex = position201, tokenIndex201 + goto l189 + l190: + position, tokenIndex = position189, tokenIndex189 if buffer[position] != rune('\\') { - goto l207 + goto l195 } position++ if buffer[position] != rune('n') { - goto l207 + goto l195 } position++ - goto l201 - l207: - position, tokenIndex = position201, tokenIndex201 + goto l189 + l195: + position, tokenIndex = position189, tokenIndex189 if buffer[position] != rune('\\') { - goto l208 + goto l196 } position++ if buffer[position] != rune('"') { - goto l208 + goto l196 } position++ - goto l201 - l208: - position, tokenIndex = position201, tokenIndex201 + goto l189 + l196: + position, tokenIndex = position189, tokenIndex189 if buffer[position] != rune('\\') { - goto l209 + goto l197 } position++ if buffer[position] != rune('\'') { - goto l209 + goto l197 } position++ - goto l201 - l209: - position, tokenIndex = position201, tokenIndex201 + goto l189 + l197: + position, tokenIndex = position189, tokenIndex189 if buffer[position] != rune('\\') { - goto l200 + goto l188 } position++ if buffer[position] != rune('\\') { - goto l200 + goto l188 } position++ } - l201: - goto l199 - l200: - position, tokenIndex = position200, tokenIndex200 + l189: + goto l187 + l188: + position, tokenIndex = position188, tokenIndex188 } - add(ruledoublequotedstring, position198) + add(ruledoublequotedstring, position186) } return true }, /* 15 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ - nil, + func() bool { + { + position199 := position + l200: + { + position201, tokenIndex201 := position, tokenIndex + { + position202, tokenIndex202 := position, tokenIndex + { + position204, tokenIndex204 := position, tokenIndex + { + position205, tokenIndex205 := position, tokenIndex + if buffer[position] != rune('\'') { + goto l206 + } + position++ + goto l205 + l206: + position, tokenIndex = position205, tokenIndex205 + if buffer[position] != rune('\\') { + goto l207 + } + position++ + goto l205 + l207: + position, tokenIndex = position205, tokenIndex205 + if buffer[position] != rune('\n') { + goto l204 + } + position++ + } + l205: + goto l203 + l204: + position, tokenIndex = position204, tokenIndex204 + } + if !matchDot() { + goto l203 + } + goto l202 + l203: + position, tokenIndex = position202, tokenIndex202 + if buffer[position] != rune('\\') { + goto l208 + } + position++ + if buffer[position] != rune('n') { + goto l208 + } + position++ + goto l202 + l208: + position, tokenIndex = position202, tokenIndex202 + if buffer[position] != rune('\\') { + goto l209 + } + position++ + if buffer[position] != rune('"') { + goto l209 + } + position++ + goto l202 + l209: + position, tokenIndex = position202, tokenIndex202 + if buffer[position] != rune('\\') { + goto l210 + } + position++ + if buffer[position] != rune('\'') { + goto l210 + } + position++ + goto l202 + l210: + position, tokenIndex = position202, tokenIndex202 + if buffer[position] != rune('\\') { + goto l201 + } + position++ + if buffer[position] != rune('\\') { + goto l201 + } + position++ + } + l202: + goto l200 + l201: + position, tokenIndex = position201, tokenIndex201 + } + add(rulesinglequotedstring, position199) + } + return true + }, /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { position211, tokenIndex211 := position, tokenIndex @@ -2438,7 +2447,7 @@ func (p *PQL) Init() { }, /* 21 uintrow <- <( Action40)> */ nil, - /* 22 col <- <(( Action41) / ('"' '"' Action42))> */ + /* 22 col <- <(( Action41) / ('\'' '\'' Action42) / ('"' '"' Action43))> */ func() bool { position247, tokenIndex247 := position, tokenIndex { @@ -2458,24 +2467,45 @@ func (p *PQL) Init() { goto l249 l250: position, tokenIndex = position249, tokenIndex249 - if buffer[position] != rune('"') { - goto l247 + if buffer[position] != rune('\'') { + goto l253 } position++ { - position253 := position - if !_rules[ruledoublequotedstring]() { - goto l247 + position254 := position + if !_rules[rulesinglequotedstring]() { + goto l253 } - add(rulePegText, position253) + add(rulePegText, position254) } - if buffer[position] != rune('"') { - goto l247 + if buffer[position] != rune('\'') { + goto l253 } position++ { add(ruleAction42, position) } + goto l249 + l253: + position, tokenIndex = position249, tokenIndex249 + if buffer[position] != rune('"') { + goto l247 + } + position++ + { + position256 := position + if !_rules[ruledoublequotedstring]() { + goto l247 + } + add(rulePegText, position256) + } + if buffer[position] != rune('"') { + goto l247 + } + position++ + { + add(ruleAction43, position) + } } l249: add(rulecol, position248) @@ -2487,99 +2517,99 @@ func (p *PQL) Init() { }, /* 23 open <- <('(' sp)> */ func() bool { - position255, tokenIndex255 := position, tokenIndex + position258, tokenIndex258 := position, tokenIndex { - position256 := position + position259 := position if buffer[position] != rune('(') { - goto l255 + goto l258 } position++ if !_rules[rulesp]() { - goto l255 + goto l258 } - add(ruleopen, position256) + add(ruleopen, position259) } return true - l255: - position, tokenIndex = position255, tokenIndex255 + l258: + position, tokenIndex = position258, tokenIndex258 return false }, /* 24 close <- <(')' sp)> */ func() bool { - position257, tokenIndex257 := position, tokenIndex + position260, tokenIndex260 := position, tokenIndex { - position258 := position + position261 := position if buffer[position] != rune(')') { - goto l257 + goto l260 } position++ if !_rules[rulesp]() { - goto l257 + goto l260 } - add(ruleclose, position258) + add(ruleclose, position261) } return true - l257: - position, tokenIndex = position257, tokenIndex257 + l260: + position, tokenIndex = position260, tokenIndex260 return false }, /* 25 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position260 := position - l261: + position263 := position + l264: { - position262, tokenIndex262 := position, tokenIndex + position265, tokenIndex265 := position, tokenIndex { - position263, tokenIndex263 := position, tokenIndex + position266, tokenIndex266 := position, tokenIndex if buffer[position] != rune(' ') { - goto l264 + goto l267 } position++ - goto l263 - l264: - position, tokenIndex = position263, tokenIndex263 + goto l266 + l267: + position, tokenIndex = position266, tokenIndex266 if buffer[position] != rune('\t') { + goto l268 + } + position++ + goto l266 + l268: + position, tokenIndex = position266, tokenIndex266 + if buffer[position] != rune('\n') { goto l265 } position++ - goto l263 - l265: - position, tokenIndex = position263, tokenIndex263 - if buffer[position] != rune('\n') { - goto l262 - } - position++ } - l263: - goto l261 - l262: - position, tokenIndex = position262, tokenIndex262 + l266: + goto l264 + l265: + position, tokenIndex = position265, tokenIndex265 } - add(rulesp, position260) + add(rulesp, position263) } return true }, /* 26 comma <- <(sp ',' sp)> */ func() bool { - position266, tokenIndex266 := position, tokenIndex + position269, tokenIndex269 := position, tokenIndex { - position267 := position + position270 := position if !_rules[rulesp]() { - goto l266 + goto l269 } if buffer[position] != rune(',') { - goto l266 + goto l269 } position++ if !_rules[rulesp]() { - goto l266 + goto l269 } - add(rulecomma, position267) + add(rulecomma, position270) } return true - l266: - position, tokenIndex = position266, tokenIndex266 + l269: + position, tokenIndex = position269, tokenIndex269 return false }, /* 27 lbrack <- <('[' sp)> */ @@ -2590,139 +2620,139 @@ func (p *PQL) Init() { nil, /* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position271, tokenIndex271 := position, tokenIndex + position274, tokenIndex274 := position, tokenIndex { - position272 := position + position275 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l271 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l271 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l271 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l271 + goto l274 } position++ if buffer[position] != rune('-') { - goto l271 + goto l274 } position++ { - position273, tokenIndex273 := position, tokenIndex + position276, tokenIndex276 := position, tokenIndex if buffer[position] != rune('0') { + goto l277 + } + position++ + goto l276 + l277: + position, tokenIndex = position276, tokenIndex276 + if buffer[position] != rune('1') { goto l274 } position++ - goto l273 - l274: - position, tokenIndex = position273, tokenIndex273 - if buffer[position] != rune('1') { - goto l271 - } - position++ } - l273: + l276: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l271 + goto l274 } position++ if buffer[position] != rune('-') { - goto l271 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l271 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l271 + goto l274 } position++ if buffer[position] != rune('T') { - goto l271 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l271 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l271 + goto l274 } position++ if buffer[position] != rune(':') { - goto l271 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l271 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l271 + goto l274 } position++ - add(ruletimestampbasicfmt, position272) + add(ruletimestampbasicfmt, position275) } return true - l271: - position, tokenIndex = position271, tokenIndex271 + l274: + position, tokenIndex = position274, tokenIndex274 return false }, /* 31 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ func() bool { - position275, tokenIndex275 := position, tokenIndex + position278, tokenIndex278 := position, tokenIndex { - position276 := position + position279 := position { - position277, tokenIndex277 := position, tokenIndex + position280, tokenIndex280 := position, tokenIndex if buffer[position] != rune('"') { - goto l278 + goto l281 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l278 + goto l281 } if buffer[position] != rune('"') { + goto l281 + } + position++ + goto l280 + l281: + position, tokenIndex = position280, tokenIndex280 + if buffer[position] != rune('\'') { + goto l282 + } + position++ + if !_rules[ruletimestampbasicfmt]() { + goto l282 + } + if buffer[position] != rune('\'') { + goto l282 + } + position++ + goto l280 + l282: + position, tokenIndex = position280, tokenIndex280 + if !_rules[ruletimestampbasicfmt]() { goto l278 } - position++ - goto l277 - l278: - position, tokenIndex = position277, tokenIndex277 - if buffer[position] != rune('\'') { - goto l279 - } - position++ - if !_rules[ruletimestampbasicfmt]() { - goto l279 - } - if buffer[position] != rune('\'') { - goto l279 - } - position++ - goto l277 - l279: - position, tokenIndex = position277, tokenIndex277 - if !_rules[ruletimestampbasicfmt]() { - goto l275 - } } - l277: - add(ruletimestampfmt, position276) + l280: + add(ruletimestampfmt, position279) } return true - l275: - position, tokenIndex = position275, tokenIndex275 + l278: + position, tokenIndex = position278, tokenIndex278 return false }, - /* 32 timestamp <- <( Action43)> */ + /* 32 timestamp <- <( Action44)> */ nil, /* 34 Action0 <- <{p.startCall("Set")}> */ nil, @@ -2811,7 +2841,9 @@ func (p *PQL) Init() { nil, /* 77 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 78 Action43 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 78 Action43 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + nil, + /* 79 Action44 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index ad40364b3..076261599 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -68,6 +68,14 @@ func TestPEGWorking(t *testing.T) { name: "Set", input: "Set(2, f=10)", ncalls: 1}, + { + name: "SetWithColKeySingleQuote", + input: `Set('foo', f=10)`, + ncalls: 1}, + { + name: "SetWithColKeyDoubleQuote", + input: `Set("foo", f=10)`, + ncalls: 1}, { name: "SetTime", input: "Set(2, f=1, 1999-12-31T00:00)", diff --git a/server.go b/server.go index 91c2d813a..db577fb86 100644 --- a/server.go +++ b/server.go @@ -369,17 +369,28 @@ func (s *Server) Close() error { close(s.closing) s.wg.Wait() + var errh error + var errt error + var errc error if s.cluster != nil { - s.cluster.close() + errc = s.cluster.close() } if s.holder != nil { - s.holder.Close() + errh = s.holder.Close() } if s.translateFile != nil { - s.translateFile.Close() + errt = s.translateFile.Close() } - - return nil + // prefer to return holder error over translateFile error over cluster + // error. This order is somewhat arbitrary. It would be better if we had + // some way to combine all the errors, but probably not important enough to + // warrant the extra complexity. + if errh != nil { + return errors.Wrap(errh, "closing holder") + } else if errt != nil { + return errors.Wrap(errt, "closing translateFile") + } + return errors.Wrap(errc, "closing cluster") } // loadNodeID gets NodeID from disk, or creates a new value. diff --git a/server/server.go b/server/server.go index d71139096..9060256af 100644 --- a/server/server.go +++ b/server/server.go @@ -20,7 +20,7 @@ package server import ( - "fmt" + "crypto/tls" "io" "log" "math/rand" @@ -31,7 +31,7 @@ import ( "syscall" "time" - "crypto/tls" + "golang.org/x/sync/errgroup" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" @@ -62,6 +62,7 @@ type Command struct { // Gossip transport gossipTransport *gossip.Transport + gossipMemberSet io.Closer // Standard input/output *pilosa.CmdIO @@ -75,9 +76,10 @@ type Command struct { logOutput io.Writer logger loggerLogger - Handler pilosa.Handler - API *pilosa.API - ln net.Listener + Handler pilosa.Handler + API *pilosa.API + ln net.Listener + closeTimeout time.Duration serverOptions []pilosa.ServerOption } @@ -91,6 +93,13 @@ func OptCommandServerOptions(opts ...pilosa.ServerOption) CommandOption { } } +func OptCommandCloseTimeout(d time.Duration) CommandOption { + return func(c *Command) error { + c.closeTimeout = d + return nil + } +} + // NewCommand returns a new instance of Main. func NewCommand(stdin io.Reader, stdout, stderr io.Writer, opts ...CommandOption) *Command { c := &Command{ @@ -294,6 +303,7 @@ func (m *Command) SetupServer() error { http.OptHandlerAPI(m.API), http.OptHandlerLogger(m.logger), http.OptHandlerListener(m.ln), + http.OptHandlerCloseTimeout(m.closeTimeout), ) return errors.Wrap(err, "new handler") @@ -317,7 +327,7 @@ func (m *Command) setupNetworking() error { return errors.Wrap(err, "getting transport") } - gossipMemberSet, err := gossip.NewGossipMemberSet( + gossipMemberSet, err := gossip.NewMemberSet( m.Config.Gossip, m.API, gossip.WithLogger(m.logger.Logger()), @@ -326,6 +336,8 @@ func (m *Command) setupNetworking() error { if err != nil { return errors.Wrap(err, "getting memberset") } + m.gossipMemberSet = gossipMemberSet + return errors.Wrap(gossipMemberSet.Open(), "opening gossip memberset") } @@ -338,17 +350,18 @@ func (m *Command) GossipTransport() *gossip.Transport { // Close shuts down the server. func (m *Command) Close() error { - var logErr error - handlerErr := m.Handler.Close() - serveErr := m.Server.Close() + defer close(m.done) + eg := errgroup.Group{} + eg.Go(m.Handler.Close) + eg.Go(m.Server.Close) + if m.gossipMemberSet != nil { + eg.Go(m.gossipMemberSet.Close) + } if closer, ok := m.logOutput.(io.Closer); ok { - logErr = closer.Close() + eg.Go(closer.Close) } - close(m.done) - if serveErr != nil || logErr != nil || handlerErr != nil { - return fmt.Errorf("closing server: '%v', closing logs: '%v', closing handler: '%v'", serveErr, logErr, handlerErr) - } - return nil + err := eg.Wait() + return errors.Wrap(err, "closing everything") } // newStatsClient creates a stats client from the config diff --git a/test/pilosa.go b/test/pilosa.go index e1022ce8b..001949906 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -23,6 +23,7 @@ import ( "os" "strings" "testing" + "time" "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" @@ -55,6 +56,11 @@ func newCommand(opts ...server.CommandOption) *Command { panic(err) } + // set aggressive close timeout by default to avoid hanging tests. This was + // a problem with PDK tests which used go-pilosa as well. We put it at the + // beginning of the option slice so that it can be overridden by user-passed + // options. + opts = append([]server.CommandOption{server.OptCommandCloseTimeout(time.Millisecond * 2)}, opts...) m := &Command{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr, opts...), commandOptions: opts} m.Config.DataDir = path m.Config.Bind = "http://localhost:0" diff --git a/translate_mapsize_386.go b/translate_mapsize_386.go index b8beafa13..259b735ac 100644 --- a/translate_mapsize_386.go +++ b/translate_mapsize_386.go @@ -1,5 +1,5 @@ package pilosa -// DefaultMapSize is the default size of mapped memory for the translate store. +// defaultMapSize is the default size of mapped memory for the translate store. // It is passed as an int to syscall.Mmap and so must be < 2^31 -const DefaultMapSize = (1 << 31) - 1 // 2GB +const defaultMapSize = (1 << 31) - 1 // 2GB