diff --git a/.travis.yml b/.travis.yml index 937416af0..fe4a2b8f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,6 @@ go: - 1.8 - master addons: - apt: - sources: - - sourceline: 'ppa:masterminds/glide' - packages: - - glide before_install: - go get github.com/mattn/goveralls script: diff --git a/Dockerfile b/Dockerfile index d49baaf8c..7a8cd4a8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,10 @@ -FROM golang:1.8.1 as builder +FROM golang:1.8.3 as builder ARG ldflags='' -ARG GLIDE="https://github.com/Masterminds/glide/releases/download/v0.12.3/glide-v0.12.3-linux-amd64.tar.gz" -ARG GLIDE_HASH="d6d3816c70fba716466e7381a9c06cb31565a3b87acb5bad9dd3beb0a9f9b0f8" COPY . /go/src/github.com/pilosa/pilosa -RUN wget ${GLIDE} -O /go/glide.tar.gz -q \ - && tar xf /go/glide.tar.gz \ - && mv /go/linux-amd64/glide /go/bin \ - && echo "${GLIDE_HASH} /go/bin/glide" | shasum -a 256 -c - \ - && cd /go/src/github.com/pilosa/pilosa \ +RUN cd /go/src/github.com/pilosa/pilosa \ && make vendor \ && CGO_ENABLED=0 go install -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 000000000..17ba49821 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,199 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/BurntSushi/toml" + packages = ["."] + revision = "99064174e013895bbd9b025c31100bd1d9b590ca" + +[[projects]] + branch = "master" + name = "github.com/CAFxX/gcnotifier" + packages = ["."] + revision = "adea3e70515666981da25214a7d3e377e4841c22" + +[[projects]] + name = "github.com/DataDog/datadog-go" + packages = ["statsd"] + revision = "909c02b65dd8a52e8fa6072db9752a112227cf21" + version = "1.0.0" + +[[projects]] + name = "github.com/armon/go-metrics" + packages = ["."] + revision = "97c69685293dce4c0a2d0b19535179bbc976e4d2" + +[[projects]] + name = "github.com/boltdb/bolt" + packages = ["."] + revision = "4b1ebc1869ad66568b313d0dc410e2be72670dda" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + name = "github.com/fsnotify/fsnotify" + packages = ["."] + revision = "7d7316ed6e1ed2de075aab8dfc76de5d158d66e1" + +[[projects]] + name = "github.com/gogo/protobuf" + packages = ["proto"] + revision = "a9cd0c35b97daf74d0ebf3514c5254814b2703b4" + +[[projects]] + name = "github.com/golang/groupcache" + packages = ["lru"] + revision = "a6b377e3400b08991b80d6805d627f347f983866" + +[[projects]] + branch = "master" + name = "github.com/golang/protobuf" + packages = ["proto"] + revision = "8ee79997227bf9b34611aee7946ae64735e6fd93" + +[[projects]] + branch = "master" + name = "github.com/gorilla/context" + packages = ["."] + revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" + +[[projects]] + name = "github.com/gorilla/mux" + packages = ["."] + revision = "392c28fe23e1c45ddba891b0320b3b5df220beea" + version = "v1.3.0" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/errwrap" + packages = ["."] + revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/go-msgpack" + packages = ["codec"] + revision = "fa3f63826f7c23912c15263591e65d54d080b458" + +[[projects]] + name = "github.com/hashicorp/go-multierror" + packages = ["."] + revision = "ed905158d87462226a13fe39ddf685ea65f1c11f" + +[[projects]] + name = "github.com/hashicorp/hcl" + packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] + revision = "630949a3c5fa3c613328e1b8256052cbc2327c9b" + +[[projects]] + name = "github.com/hashicorp/memberlist" + packages = ["."] + revision = "9800c50ab79c002353852a9b1095e9591b161513" + version = "v0.1.0" + +[[projects]] + branch = "master" + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + +[[projects]] + name = "github.com/magiconair/properties" + packages = ["."] + revision = "b3b15ef068fd0b17ddf408a23669f20811d194d2" + +[[projects]] + name = "github.com/miekg/dns" + packages = ["."] + revision = "ca336a1f95a6b89be9c250df26c7a41742eb4a6f" + +[[projects]] + name = "github.com/mitchellh/mapstructure" + packages = ["."] + revision = "db1efb556f84b25a0a13a04aad883943538ad2e0" + +[[projects]] + name = "github.com/pelletier/go-buffruneio" + packages = ["."] + revision = "c37440a7cf42ac63b919c752ca73a85067e05992" + version = "v0.2.0" + +[[projects]] + name = "github.com/pelletier/go-toml" + packages = ["."] + revision = "23f644976aa7c724adf4aec911dadf4af17840ab" + +[[projects]] + name = "github.com/rakyll/statik" + packages = ["fs"] + revision = "89fe3459b5c829c32e89bdff9c43f18aad728f2f" + +[[projects]] + branch = "master" + name = "github.com/spf13/afero" + packages = [".","mem"] + revision = "9be650865eab0c12963d8753212f4f9c66cdcf12" + +[[projects]] + name = "github.com/spf13/cast" + packages = ["."] + revision = "4f1683a2242a92e62d6ff705a30e435cbf2b50a3" + +[[projects]] + branch = "master" + name = "github.com/spf13/cobra" + packages = ["."] + revision = "fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c" + +[[projects]] + name = "github.com/spf13/jwalterweatherman" + packages = ["."] + revision = "fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "9ff6c6923cfffbcd502984b8e0c80539a94968b7" + +[[projects]] + branch = "master" + name = "github.com/spf13/viper" + packages = ["."] + revision = "7538d73b4eb9511d85a9f1dfef202eeb8ac260f4" + +[[projects]] + name = "golang.org/x/net" + packages = ["context"] + revision = "60c41d1de8da134c05b7b40154a9a82bf5b7edb9" + +[[projects]] + branch = "master" + name = "golang.org/x/sync" + packages = ["errgroup"] + revision = "450f422ab23cf9881c94e2db30cac0eb1b7cf80c" + +[[projects]] + name = "golang.org/x/sys" + packages = ["unix"] + revision = "c200b10b5d5e122be351b67af224adc6128af5bf" + +[[projects]] + name = "golang.org/x/text" + packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"] + revision = "5a42fa2464759cbb7ee0af9de00b54d69f09a29c" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "a3f3340b5840cee44f372bddb5880fcbc419b46a" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "84ff0992f3a6023a9d4832d6aea43d6aec19878a4b9e991ba8ff8269b589e816" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 000000000..b3e45f428 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,49 @@ + +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" + +[[constraint]] + name = "github.com/BurntSushi/toml" + revision = "99064174e013895bbd9b025c31100bd1d9b590ca" + +[[constraint]] + branch = "master" + name = "github.com/CAFxX/gcnotifier" + +[[constraint]] + name = "github.com/boltdb/bolt" + revision = "4b1ebc1869ad66568b313d0dc410e2be72670dda" + +[[constraint]] + name = "github.com/gorilla/mux" + version = "1.3.0" + +[[constraint]] + name = "github.com/hashicorp/memberlist" + version = "=0.1.0" + +[[constraint]] + branch = "master" + name = "github.com/spf13/cobra" + +[[constraint]] + branch = "master" + name = "github.com/spf13/viper" diff --git a/Makefile b/Makefile index d26d6b42c..b064c7ce0 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release test cover cover-pkg cover-viz +.PHONY: dep docker pilosa crossbuild install generate statik release test cover cover-pkg cover-viz -GLIDE := $(shell command -v glide 2>/dev/null) +DEP := $(shell command -v dep 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) PROTOC := $(shell command -v protoc 2>/dev/null) VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -15,24 +15,18 @@ default: test pilosa $(GOPATH)/bin: mkdir $(GOPATH)/bin -glide: $(GOPATH)/bin -ifndef GLIDE - curl https://glide.sh/get | sh +dep: $(GOPATH)/bin + go get -u github.com/golang/dep/cmd/dep + +vendor: Gopkg.toml +ifndef DEP + make dep endif + dep ensure + touch vendor -$(GLIDE): - make glide - -vendor: $(GLIDE) glide.yaml -ifndef GLIDE - curl https://glide.sh/get | sh -endif - glide install - -glide.lock: glide glide.yaml - glide update - -vendor-update: glide.lock +Gopkg.lock: dep Gopkg.toml + dep ensure test: vendor go test $(PKGS) $(TESTFLAGS) diff --git a/README-dev.md b/README-dev.md index c5fe72368..809befbc3 100644 --- a/README-dev.md +++ b/README-dev.md @@ -21,12 +21,18 @@ git clone git@github.com:${USER}/pilosa.git cd ${GOPATH}/src/github.com/pilosa/pilosa ``` -Install [Glide][] to manage dependencies. +Install `dep` to manage dependencies: + +```sh +go get -u github.com/golang/dep/cmd/dep +``` Install Pilosa command line tools: ```sh -go install github.com/pilosa/pilosa/cmd/... +make install +# or: +# dep ensure && go install github.com/pilosa/pilosa/cmd/... ``` Running `pilosa` should now run a Pilosa instance. @@ -61,5 +67,3 @@ git push --set-upstream origin a-branch-for-the-task ``` All left to do is creating a pull request on github.com. - -[Glide]: http://glide.sh/ diff --git a/cache.go b/cache.go index c32881a29..242e5a6ab 100644 --- a/cache.go +++ b/cache.go @@ -54,7 +54,7 @@ type Cache interface { SetStats(s StatsClient) } -// LRUCache represents a least recently used Cache implemenation. +// LRUCache represents a least recently used Cache implementation. type LRUCache struct { cache *lru.Cache counts map[uint64]uint64 @@ -483,3 +483,35 @@ func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) { func (s *SimpleCache) Add(id uint64, b *Bitmap) { s.cache[id] = b } + +// NopCache represents a no-op Cache implementation. +type NopCache struct { + stats StatsClient +} + +// Ensure NopCache implements Cache. +var _ Cache = &NopCache{} + +// NewNopCache returns a new instance of NopCache. +func NewNopCache() *NopCache { + return &NopCache{ + stats: NopStatsClient, + } +} + +func (c *NopCache) Add(id uint64, n uint64) {} +func (c *NopCache) BulkAdd(id uint64, n uint64) {} +func (c *NopCache) Get(id uint64) uint64 { return 0 } +func (c *NopCache) IDs() []uint64 { return make([]uint64, 0, 0) } + +func (c *NopCache) Invalidate() {} +func (c *NopCache) Len() int { return 0 } +func (c *NopCache) Recalculate() { +} +func (c *NopCache) SetStats(s StatsClient) { + c.stats = s +} + +func (c *NopCache) Top() []BitmapPair { + return []BitmapPair{} +} diff --git a/config.go b/config.go index 30c1ebdd3..3b248878a 100644 --- a/config.go +++ b/config.go @@ -14,9 +14,7 @@ package pilosa -import ( - "time" -) +import "time" // Cluster types. const ( @@ -51,8 +49,9 @@ var ClusterTypes = []string{ClusterNone, ClusterStatic, ClusterHTTP, ClusterGoss // Config represents the configuration for the command. type Config struct { - DataDir string `toml:"data-dir"` - Bind string `toml:"bind"` + DataDir string `toml:"data-dir"` + Bind string `toml:"bind"` + InternalPort string `toml:"internal-port"` Cluster struct { ReplicaN int `toml:"replicas"` @@ -60,7 +59,6 @@ type Config struct { Hosts []string `toml:"hosts"` InternalHosts []string `toml:"internal-hosts"` PollInterval Duration `toml:"poll-interval"` - InternalPort string `toml:"internal-port"` GossipSeed string `toml:"gossip-seed"` LongQueryTime Duration `toml:"long-query-time"` } `toml:"cluster"` @@ -114,19 +112,24 @@ func (c *Config) Validate() error { if c.Cluster.ReplicaN > len(c.Cluster.Hosts) { return ErrConfigReplicaNInvalid } - if len(c.Cluster.Hosts) != len(c.Cluster.InternalHosts) { - return ErrConfigHostsMismatch - } if !foundItem(c.Cluster.Hosts, c.Bind) { return ErrConfigHostsMissing } - if !ContainsSubstring(c.Cluster.InternalPort, c.Cluster.InternalHosts) { + } + if c.Cluster.Type == ClusterHTTP { + if len(c.Cluster.Hosts) != len(c.Cluster.InternalHosts) { + return ErrConfigHostsMismatch + } + // TODO: this seems like an odd check; it's just ensuring that InternalPort + // matches any one substring from any of the InternalHosts. + // I suggest we either remove this completely or make it actually check + // the port portion of the address for this node. (note that this only applies + // to the http broadcaster, so if we simply use gossip for all implementations + // we can remove this). + if !ContainsSubstring(c.InternalPort, c.Cluster.InternalHosts) { return ErrConfigBroadcastPort } } - if c.Cluster.Type == ClusterGossip && !StringInSlice(c.Cluster.GossipSeed, c.Cluster.InternalHosts) { - return ErrConfigGossipSeed - } return nil } diff --git a/config_test.go b/config_test.go index 78c4a6127..10febfa05 100644 --- a/config_test.go +++ b/config_test.go @@ -26,7 +26,7 @@ func Test_NewConfig(t *testing.T) { t.Fatal(err) } - c.Cluster.InternalPort = pilosa.DefaultInternalPort + c.InternalPort = pilosa.DefaultInternalPort c.Cluster.InternalHosts = []string{"localhost:14004", "localhost:14001"} if err := c.Validate(); err != pilosa.ErrConfigBroadcastPort { t.Fatal(err) @@ -47,11 +47,6 @@ func Test_NewConfig(t *testing.T) { c.Cluster.ReplicaN = 2 c.Cluster.Type = pilosa.ClusterGossip - c.Cluster.GossipSeed = "localhost:10101" - if err := c.Validate(); err != pilosa.ErrConfigGossipSeed { - t.Fatal(err) - } - c.Cluster.GossipSeed = "localhost:14000" if err := c.Validate(); err != nil { t.Fatal(err) diff --git a/ctl/server.go b/ctl/server.go index cde11424b..70f43bc41 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -26,6 +26,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags := cmd.Flags() flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") flags.StringVarP(&srv.Config.Bind, "bind", "b", ":10101", "Default URI on which pilosa should listen.") + flags.StringVarP(&srv.Config.InternalPort, "internal-port", "", "", "Port to which pilosa should bind for internal state sharing.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") @@ -39,7 +40,6 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]") flags.StringVarP(&srv.Config.Cluster.GossipSeed, "cluster.gossip-seed", "", "", "Host with which to seed the gossip membership.") - flags.StringVarP(&srv.Config.Cluster.InternalPort, "cluster.internal-port", "", "", "Port to which pilosa should bind for internal state sharing.") flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.") flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.") flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.") diff --git a/ctl/server_test.go b/ctl/server_test.go index b39c894b1..16dd26916 100644 --- a/ctl/server_test.go +++ b/ctl/server_test.go @@ -16,9 +16,10 @@ package ctl import ( "bytes" + "testing" + "github.com/pilosa/pilosa/server" "github.com/spf13/cobra" - "testing" ) func TestBuildServerFlags(t *testing.T) { @@ -27,8 +28,8 @@ func TestBuildServerFlags(t *testing.T) { stdin, stdout, stderr := GetIO(buf) Server := server.NewCommand(stdin, stdout, stderr) BuildServerFlags(cm, Server) - if cm.Flags().Lookup("cluster.internal-port").Name == "" { - t.Fatal("cluster.internal-port flag is missed ") + if cm.Flags().Lookup("internal-port").Name == "" { + t.Fatal("internal-port flag is missed ") } if cm.Flags().Lookup("data-dir").Name == "" { t.Fatal("data-dir flag is missed ") diff --git a/docs/configuration.md b/docs/configuration.md index 5eecde849..b87d546fe 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -58,6 +58,17 @@ Any flag that has a value that is a comma separated list on the command line bec bind = localhost:10101 ``` +#### Internal Port + +* Description: Port to which Pilosa should bind for internal communication. +* Flag: `--internal-port=11101` +* Env: `PILOSA_INTERNAL_PORT=11101` +* Config: + + ```toml + internal-port = 11101 + ``` + #### Cluster Hosts * Description: List of hosts in the cluster. Multiple hosts should be comma separated in the flag and env forms. @@ -82,18 +93,6 @@ Any flag that has a value that is a comma separated list on the command line bec internal-hosts = ["localhost:11101"] ``` -#### Cluster Internal Port - -* Description: Port to which Pilosa should bind for internal communication. -* Flag: `--cluster.internal-port=11101` -* Env: `PILOSA_CLUSTER.INTERNAL_PORT=11101` -* Config: - - ```toml - [cluster] - internal-port = 11101 - ``` - #### Cluster Poll Interval * Description: Polling interval for cluster. diff --git a/docs/installation.md b/docs/installation.md index 6f1795106..38c102fd4 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -133,16 +133,16 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH). * [Git](https://git-scm.com/) - * [Glide](http://glide.sh/) 2. Clone the repo: ``` go get -d github.com/pilosa/pilosa ``` -3. Build the Pilosa repo: +3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice web console into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa + make generate-statik make install ``` @@ -286,7 +286,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH). * [Git](https://git-scm.com/) - * [Glide](http://glide.sh/) 2. Clone the repo: ``` diff --git a/fragment.go b/fragment.go index 64c31434b..b20dfd9c0 100644 --- a/fragment.go +++ b/fragment.go @@ -249,6 +249,8 @@ func (f *Fragment) openCache() error { f.cache = NewRankCache(f.CacheSize) case CacheTypeLRU: f.cache = NewLRUCache(f.CacheSize) + case CacheTypeNone: + f.cache = NewNopCache() default: return ErrInvalidCacheType } @@ -702,6 +704,10 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { } func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair { + // Don't retrieve from storage if CacheTypeNone. + if f.CacheType == CacheTypeNone { + return f.cache.Top() + } // If no specific rows are requested, retrieve top rows. if len(rowIDs) == 0 { f.mu.Lock() diff --git a/fragment_test.go b/fragment_test.go index 52092eae9..b58db2a1a 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -416,6 +416,24 @@ func TestFragment_TopN_IDs(t *testing.T) { } } +// Ensure a fragment return none if CacheTypeNone is set +func TestFragment_TopN_NopCache(t *testing.T) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeNone) + defer f.Close() + + // Set bits on various rows. + f.MustSetBits(100, 1, 2, 3) + f.MustSetBits(101, 4, 5, 6, 7) + f.MustSetBits(102, 8, 9, 10, 11, 12) + + // Retrieve top rows. + if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(pairs, []pilosa.Pair{}) { + t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) + } +} + // Ensure the fragment cache limit works func TestFragment_TopN_CacheSize(t *testing.T) { slice := uint64(0) diff --git a/frame.go b/frame.go index 0093ba5c5..d102c9ea7 100644 --- a/frame.go +++ b/frame.go @@ -903,12 +903,13 @@ func (p importBitSet) Less(i, j int) bool { return p.rowIDs[i] < p.rowIDs[j] } const ( CacheTypeLRU = "lru" CacheTypeRanked = "ranked" + CacheTypeNone = "none" ) // IsValidCacheType returns true if v is a valid cache type. func IsValidCacheType(v string) bool { switch v { - case CacheTypeLRU, CacheTypeRanked: + case CacheTypeLRU, CacheTypeRanked, CacheTypeNone: return true default: return false diff --git a/glide.lock b/glide.lock deleted file mode 100644 index 45633e798..000000000 --- a/glide.lock +++ /dev/null @@ -1,110 +0,0 @@ -hash: 98b6811c335d30f9711d53472ad3e1ae51090a04751dd6dc3bcb97a54e01afb9 -updated: 2017-05-01T10:42:29.124689561-05:00 -imports: -- name: github.com/armon/go-metrics - version: 97c69685293dce4c0a2d0b19535179bbc976e4d2 -- name: github.com/boltdb/bolt - version: 4b1ebc1869ad66568b313d0dc410e2be72670dda -- name: github.com/BurntSushi/toml - version: 99064174e013895bbd9b025c31100bd1d9b590ca -- name: github.com/CAFxX/gcnotifier - version: adea3e70515666981da25214a7d3e377e4841c22 -- name: github.com/DataDog/datadog-go - version: 909c02b65dd8a52e8fa6072db9752a112227cf21 - subpackages: - - statsd -- name: github.com/davecgh/go-spew - version: 346938d642f2ec3594ed81d874461961cd0faa76 - subpackages: - - spew -- name: github.com/fsnotify/fsnotify - version: 7d7316ed6e1ed2de075aab8dfc76de5d158d66e1 -- name: github.com/gogo/protobuf - version: a9cd0c35b97daf74d0ebf3514c5254814b2703b4 - subpackages: - - proto -- name: github.com/golang/groupcache - version: a6b377e3400b08991b80d6805d627f347f983866 - subpackages: - - lru -- name: github.com/golang/protobuf - version: 8ee79997227bf9b34611aee7946ae64735e6fd93 - subpackages: - - proto -- name: github.com/gorilla/context - version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42 -- name: github.com/gorilla/mux - version: 392c28fe23e1c45ddba891b0320b3b5df220beea -- name: github.com/hashicorp/errwrap - version: 7554cd9344cec97297fa6649b055a8c98c2a1e55 -- name: github.com/hashicorp/go-msgpack - version: fa3f63826f7c23912c15263591e65d54d080b458 - subpackages: - - codec -- name: github.com/hashicorp/go-multierror - version: ed905158d87462226a13fe39ddf685ea65f1c11f -- name: github.com/hashicorp/hcl - version: 630949a3c5fa3c613328e1b8256052cbc2327c9b - subpackages: - - hcl/ast - - hcl/parser - - hcl/scanner - - hcl/strconv - - hcl/token - - json/parser - - json/scanner - - json/token -- name: github.com/hashicorp/memberlist - version: 9800c50ab79c002353852a9b1095e9591b161513 -- name: github.com/inconshreveable/mousetrap - version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 -- name: github.com/magiconair/properties - version: b3b15ef068fd0b17ddf408a23669f20811d194d2 -- name: github.com/miekg/dns - version: ca336a1f95a6b89be9c250df26c7a41742eb4a6f -- name: github.com/mitchellh/mapstructure - version: db1efb556f84b25a0a13a04aad883943538ad2e0 -- name: github.com/pelletier/go-buffruneio - version: c37440a7cf42ac63b919c752ca73a85067e05992 -- name: github.com/pelletier/go-toml - version: 23f644976aa7c724adf4aec911dadf4af17840ab -- name: github.com/rakyll/statik - version: 89fe3459b5c829c32e89bdff9c43f18aad728f2f - subpackages: - - fs -- name: github.com/satori/go.uuid - version: 879c5887cd475cd7864858769793b2ceb0d44feb -- name: github.com/spf13/afero - version: 9be650865eab0c12963d8753212f4f9c66cdcf12 - subpackages: - - mem -- name: github.com/spf13/cast - version: 4f1683a2242a92e62d6ff705a30e435cbf2b50a3 -- name: github.com/spf13/cobra - version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c -- name: github.com/spf13/jwalterweatherman - version: fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66 -- name: github.com/spf13/pflag - version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 -- name: github.com/spf13/viper - version: 7538d73b4eb9511d85a9f1dfef202eeb8ac260f4 -- name: golang.org/x/net - version: 60c41d1de8da134c05b7b40154a9a82bf5b7edb9 - subpackages: - - context -- name: golang.org/x/sync - version: 450f422ab23cf9881c94e2db30cac0eb1b7cf80c - subpackages: - - errgroup -- name: golang.org/x/sys - version: c200b10b5d5e122be351b67af224adc6128af5bf - subpackages: - - unix -- name: golang.org/x/text - version: 5a42fa2464759cbb7ee0af9de00b54d69f09a29c - subpackages: - - transform - - unicode/norm -- name: gopkg.in/yaml.v2 - version: a3f3340b5840cee44f372bddb5880fcbc419b46a -testImports: [] diff --git a/glide.yaml b/glide.yaml deleted file mode 100644 index 32421ffad..000000000 --- a/glide.yaml +++ /dev/null @@ -1,40 +0,0 @@ -package: github.com/pilosa/pilosa -import: -- package: github.com/BurntSushi/toml - version: 99064174e013895bbd9b025c31100bd1d9b590ca -- package: github.com/DataDog/datadog-go - version: ~1.0.0 - subpackages: - - statsd -- package: github.com/boltdb/bolt - version: 4b1ebc1869ad66568b313d0dc410e2be72670dda -- package: github.com/davecgh/go-spew - version: ~1.1.0 - subpackages: - - spew -- package: github.com/gogo/protobuf - version: a9cd0c35b97daf74d0ebf3514c5254814b2703b4 - subpackages: - - proto -- package: github.com/golang/groupcache - version: a6b377e3400b08991b80d6805d627f347f983866 - subpackages: - - lru -- package: golang.org/x/sys - version: c200b10b5d5e122be351b67af224adc6128af5bf - subpackages: - - unix -- package: github.com/golang/protobuf -- package: github.com/satori/go.uuid - version: ^1.1.0 -- package: github.com/spf13/cobra -- package: github.com/spf13/viper -- package: github.com/gorilla/mux - version: ^1.3.0 -- package: github.com/rakyll/statik - version: 89fe3459b5c829c32e89bdff9c43f18aad728f2f - subpackages: - - fs -- package: github.com/hashicorp/memberlist -- package: golang.org/x/sync -- package: github.com/CAFxX/gcnotifier diff --git a/handler.go b/handler.go index a69545c5f..ab7a51c06 100644 --- a/handler.go +++ b/handler.go @@ -29,6 +29,7 @@ import ( "net/http" _ "net/http/pprof" "os" + "runtime/debug" "strconv" "strings" "time" @@ -142,6 +143,16 @@ func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request // ServeHTTP handles an HTTP request. func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + stack := debug.Stack() + msg := "PANIC: %s\n%s" + fmt.Fprintf(h.LogOutput, msg, err, stack) + fmt.Fprintf(w, msg, err, stack) + } + }() + t := time.Now() h.Router.ServeHTTP(w, r) dif := time.Since(t) diff --git a/handler_test.go b/handler_test.go index f2bb02f63..6e13bf6ba 100644 --- a/handler_test.go +++ b/handler_test.go @@ -33,6 +33,30 @@ import ( "github.com/pilosa/pilosa/test" ) +func TestHandlerPanics(t *testing.T) { + h := test.NewHandler() + buf := &bytes.Buffer{} + h.Handler.LogOutput = buf + + w := httptest.NewRecorder() + // will panic since Handler has no Holder set up + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil)) + bufbytes, err := ioutil.ReadAll(buf) + if err != nil { + t.Fatalf("reading all logoutput: %v", err) + } + if !bytes.Contains(bufbytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) { + t.Fatalf("expected panic in log, but got: %s", bufbytes) + } + if w.Code != http.StatusInternalServerError { + t.Fatalf("expected internal server error, but got: %v", w.Code) + } + bodyBytes := w.Body.Bytes() + if !bytes.Contains(bodyBytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) { + t.Fatalf("response to client should have panic, but got %s", bodyBytes) + } +} + // Ensure the handler returns "not found" for invalid paths. func TestHandler_NotFound(t *testing.T) { hldr := test.MustOpenHolder() diff --git a/index.go b/index.go index 16720630e..8f1846c5f 100644 --- a/index.go +++ b/index.go @@ -414,16 +414,12 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { } // Validate mutually exclusive options if ranges are enabled. - // - // NOTE(https://github.com/pilosa/pilosa/issues/399): - // Cache type should be validated as "none" once it is allowed. if opt.RangeEnabled { if opt.InverseEnabled { return nil, ErrInverseRangeNotAllowed - } else if opt.CacheType != "" && opt.CacheType != CacheTypeLRU { + } else if opt.CacheType != "" && opt.CacheType != CacheTypeNone { return nil, ErrRangeCacheNotAllowed } - opt.CacheSize = 0 } else { if len(opt.Fields) > 0 { return nil, ErrFrameFieldsNotAllowed diff --git a/index_test.go b/index_test.go index 5188c9a59..a7cff68f7 100644 --- a/index_test.go +++ b/index_test.go @@ -152,6 +152,18 @@ func TestIndex_CreateFrame(t *testing.T) { } }) + t.Run("RangeEnabledWithCacheTypeNone", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + CacheType: pilosa.CacheTypeNone, + CacheSize: uint32(5), + }); err != nil { + t.Fatal(err) + } + }) + t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() diff --git a/pilosa.go b/pilosa.go index 9f9ca7c59..aae9c0dab 100644 --- a/pilosa.go +++ b/pilosa.go @@ -75,7 +75,6 @@ var ( ErrConfigBroadcastPort = errors.New("internal-port not found in internal-hosts") ErrConfigHostsMismatch = errors.New("hosts and internal-hosts length mismatch") ErrConfigReplicaNInvalid = errors.New("replica number must be <= hosts") - ErrConfigGossipSeed = errors.New("invalid gossip seed") ) // Regular expression to validate index and frame names. @@ -153,7 +152,7 @@ func ValidateLabel(label string) error { return nil } -// StringInSlice checks is substring a is in the slice +// StringInSlice checks for substring a in the slice. func StringInSlice(a string, list []string) bool { for _, b := range list { if b == a { @@ -163,7 +162,7 @@ func StringInSlice(a string, list []string) bool { return false } -// ContainsSubstring checks is substring a is contained in the slice +// ContainsSubstring checks to see if substring a is contained in any string in the slice. func ContainsSubstring(a string, list []string) bool { for _, b := range list { if strings.Contains(b, a) { diff --git a/server/server.go b/server/server.go index 6e51c4e74..1553917bb 100644 --- a/server/server.go +++ b/server/server.go @@ -153,8 +153,8 @@ func (m *Command) SetupServer() error { // Set internal port (string). internalPortStr := pilosa.DefaultInternalPort - if m.Config.Cluster.InternalPort != "" { - internalPortStr = m.Config.Cluster.InternalPort + if m.Config.InternalPort != "" { + internalPortStr = m.Config.InternalPort } switch m.Config.Cluster.Type {