From 7aa5f685506db2053fd44b35336faf125ddccc5a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Jul 2017 11:33:25 -0700 Subject: [PATCH 01/11] add make generate-statik to docs --- docs/installation.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 6f1795106..220e86e16 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -140,9 +140,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) go get -d github.com/pilosa/pilosa ``` -3. Build the Pilosa repo: +3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice web console into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa + make generate-statik make install ``` From 0385e601901f60531913f592dbb2be5e5d4fac9a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Jul 2017 16:41:43 -0700 Subject: [PATCH 02/11] add recovery in top level handler --- handler.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/handler.go b/handler.go index bffb36947..d6b9c4d0c 100644 --- a/handler.go +++ b/handler.go @@ -29,6 +29,7 @@ import ( "net/http" _ "net/http/pprof" "os" + "runtime/debug" "strconv" "strings" "time" @@ -138,6 +139,16 @@ func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request // ServeHTTP handles an HTTP request. func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + stack := debug.Stack() + msg := "PANIC: %s\n%s" + fmt.Fprintf(h.LogOutput, msg, err, stack) + fmt.Fprintf(w, msg, err, stack) + } + }() + t := time.Now() h.Router.ServeHTTP(w, r) dif := time.Since(t) From 24fc4a254d6a09067346600b2c61fef1d5358621 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Jul 2017 17:04:26 -0700 Subject: [PATCH 03/11] add test for panic recovery --- handler_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/handler_test.go b/handler_test.go index b96b755fc..eb3bc9da1 100644 --- a/handler_test.go +++ b/handler_test.go @@ -32,6 +32,30 @@ import ( "github.com/pilosa/pilosa/test" ) +func TestHandlerPanics(t *testing.T) { + h := test.NewHandler() + buf := &bytes.Buffer{} + h.Handler.LogOutput = buf + + w := httptest.NewRecorder() + // will panic since Handler has no Holder set up + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil)) + bufbytes, err := ioutil.ReadAll(buf) + if err != nil { + t.Fatalf("reading all logoutput: %v", err) + } + if !bytes.Contains(bufbytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) { + t.Fatalf("expected panic in log, but got: %s", bufbytes) + } + if w.Code != http.StatusInternalServerError { + t.Fatalf("expected internal server error, but got: %v", w.Code) + } + bodyBytes := w.Body.Bytes() + if !bytes.Contains(bodyBytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) { + t.Fatalf("response to client should have panic, but got %s", bodyBytes) + } +} + // Ensure the handler returns "not found" for invalid paths. func TestHandler_NotFound(t *testing.T) { hldr := test.MustOpenHolder() From f68362c40c967f2b71ae1e8034ec007175b3a450 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 20 Jul 2017 16:11:13 -0500 Subject: [PATCH 04/11] implement nopcache --- cache.go | 34 +++++++++++++++++++++++++++++++++- fragment.go | 2 ++ frame.go | 3 ++- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/cache.go b/cache.go index c32881a29..e5f1163a0 100644 --- a/cache.go +++ b/cache.go @@ -54,7 +54,7 @@ type Cache interface { SetStats(s StatsClient) } -// LRUCache represents a least recently used Cache implemenation. +// LRUCache represents a least recently used Cache implementation. type LRUCache struct { cache *lru.Cache counts map[uint64]uint64 @@ -483,3 +483,35 @@ func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) { func (s *SimpleCache) Add(id uint64, b *Bitmap) { s.cache[id] = b } + +type NopCache struct { + stats StatsClient +} + +// NopCache implement Cache interface, returns no cache for cache type None +var _ Cache = &NopCache{} + +// NewNopeCache returns a new instance of NopCache. +func NewNopCache() *NopCache { + c := &NopCache{ + stats: NopStatsClient, + } + return c +} + +func (c *NopCache) Add(id uint64, n uint64) {} +func (c *NopCache) BulkAdd(id uint64, n uint64) {} +func (c *NopCache) Get(id uint64) uint64 { return 0 } +func (c *NopCache) IDs() []uint64 { return make([]uint64, 0, 0) } + +func (c *NopCache) Invalidate() {} +func (c *NopCache) Len() int { return 0 } +func (c *NopCache) Recalculate() { +} +func (c *NopCache) SetStats(s StatsClient) { + c.stats = s +} + +func (c *NopCache) Top() []BitmapPair { + return []BitmapPair{} +} diff --git a/fragment.go b/fragment.go index 64c31434b..fdfcddd86 100644 --- a/fragment.go +++ b/fragment.go @@ -249,6 +249,8 @@ func (f *Fragment) openCache() error { f.cache = NewRankCache(f.CacheSize) case CacheTypeLRU: f.cache = NewLRUCache(f.CacheSize) + case CacheTypeNone: + f.cache = NewNopCache() default: return ErrInvalidCacheType } diff --git a/frame.go b/frame.go index 42990aeb6..d295ff01c 100644 --- a/frame.go +++ b/frame.go @@ -905,12 +905,13 @@ func (p importBitSet) Less(i, j int) bool { return p.rowIDs[i] < p.rowIDs[j] } const ( CacheTypeLRU = "lru" CacheTypeRanked = "ranked" + CacheTypeNone = "none" ) // IsValidCacheType returns true if v is a valid cache type. func IsValidCacheType(v string) bool { switch v { - case CacheTypeLRU, CacheTypeRanked: + case CacheTypeLRU, CacheTypeRanked, CacheTypeNone: return true default: return false From 2cfcf497e110aa8e660015cbd3ae1bd86c551f59 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 21 Jul 2017 00:16:37 -0500 Subject: [PATCH 05/11] add tests for none cache --- cache.go | 6 +++--- fragment.go | 4 ++++ fragment_test.go | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/cache.go b/cache.go index e5f1163a0..d6d4174cd 100644 --- a/cache.go +++ b/cache.go @@ -501,11 +501,11 @@ func NewNopCache() *NopCache { func (c *NopCache) Add(id uint64, n uint64) {} func (c *NopCache) BulkAdd(id uint64, n uint64) {} -func (c *NopCache) Get(id uint64) uint64 { return 0 } -func (c *NopCache) IDs() []uint64 { return make([]uint64, 0, 0) } +func (c *NopCache) Get(id uint64) uint64 { return 0 } +func (c *NopCache) IDs() []uint64 { return make([]uint64, 0, 0) } func (c *NopCache) Invalidate() {} -func (c *NopCache) Len() int { return 0 } +func (c *NopCache) Len() int { return 0 } func (c *NopCache) Recalculate() { } func (c *NopCache) SetStats(s StatsClient) { diff --git a/fragment.go b/fragment.go index fdfcddd86..16ec5869f 100644 --- a/fragment.go +++ b/fragment.go @@ -704,6 +704,10 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { } func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair { + // Don't retrieve from storage if CacheTypeNone + if f.CacheType == CacheTypeNone { + return f.cache.Top() + } // If no specific rows are requested, retrieve top rows. if len(rowIDs) == 0 { f.mu.Lock() diff --git a/fragment_test.go b/fragment_test.go index 01f4a46bc..5a58acd60 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -415,6 +415,24 @@ func TestFragment_TopN_IDs(t *testing.T) { } } +// Ensure a fragment can return top rows when specified by ID. +func TestFragment_TopN_NopCache(t *testing.T) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeNone) + defer f.Close() + + // Set bits on various rows. + f.MustSetBits(100, 1, 2, 3) + f.MustSetBits(101, 4, 5, 6, 7) + f.MustSetBits(102, 8, 9, 10, 11, 12) + + // Retrieve top rows. + if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(pairs, []pilosa.Pair{}) { + t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) + } +} + // Ensure the fragment cache limit works func TestFragment_TopN_CacheSize(t *testing.T) { slice := uint64(0) From 5bc9c4361198f55eb998d12e01123bfc67a91adf Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 21 Jul 2017 11:07:00 -0500 Subject: [PATCH 06/11] Switch from glide to dep for dependency management. Fixes #732. This implements the new `dep` command into our build process and removes all references to Glide. --- .travis.yml | 5 -- Dockerfile | 10 +-- Gopkg.lock | 199 +++++++++++++++++++++++++++++++++++++++++++ Gopkg.toml | 49 +++++++++++ Makefile | 29 +++---- README-dev.md | 12 ++- docs/installation.md | 2 - glide.lock | 110 ------------------------ glide.yaml | 40 --------- 9 files changed, 269 insertions(+), 187 deletions(-) create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml delete mode 100644 glide.lock delete mode 100644 glide.yaml diff --git a/.travis.yml b/.travis.yml index 937416af0..fe4a2b8f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,6 @@ go: - 1.8 - master addons: - apt: - sources: - - sourceline: 'ppa:masterminds/glide' - packages: - - glide before_install: - go get github.com/mattn/goveralls script: diff --git a/Dockerfile b/Dockerfile index d49baaf8c..7a8cd4a8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,10 @@ -FROM golang:1.8.1 as builder +FROM golang:1.8.3 as builder ARG ldflags='' -ARG GLIDE="https://github.com/Masterminds/glide/releases/download/v0.12.3/glide-v0.12.3-linux-amd64.tar.gz" -ARG GLIDE_HASH="d6d3816c70fba716466e7381a9c06cb31565a3b87acb5bad9dd3beb0a9f9b0f8" COPY . /go/src/github.com/pilosa/pilosa -RUN wget ${GLIDE} -O /go/glide.tar.gz -q \ - && tar xf /go/glide.tar.gz \ - && mv /go/linux-amd64/glide /go/bin \ - && echo "${GLIDE_HASH} /go/bin/glide" | shasum -a 256 -c - \ - && cd /go/src/github.com/pilosa/pilosa \ +RUN cd /go/src/github.com/pilosa/pilosa \ && make vendor \ && CGO_ENABLED=0 go install -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 000000000..17ba49821 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,199 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/BurntSushi/toml" + packages = ["."] + revision = "99064174e013895bbd9b025c31100bd1d9b590ca" + +[[projects]] + branch = "master" + name = "github.com/CAFxX/gcnotifier" + packages = ["."] + revision = "adea3e70515666981da25214a7d3e377e4841c22" + +[[projects]] + name = "github.com/DataDog/datadog-go" + packages = ["statsd"] + revision = "909c02b65dd8a52e8fa6072db9752a112227cf21" + version = "1.0.0" + +[[projects]] + name = "github.com/armon/go-metrics" + packages = ["."] + revision = "97c69685293dce4c0a2d0b19535179bbc976e4d2" + +[[projects]] + name = "github.com/boltdb/bolt" + packages = ["."] + revision = "4b1ebc1869ad66568b313d0dc410e2be72670dda" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + name = "github.com/fsnotify/fsnotify" + packages = ["."] + revision = "7d7316ed6e1ed2de075aab8dfc76de5d158d66e1" + +[[projects]] + name = "github.com/gogo/protobuf" + packages = ["proto"] + revision = "a9cd0c35b97daf74d0ebf3514c5254814b2703b4" + +[[projects]] + name = "github.com/golang/groupcache" + packages = ["lru"] + revision = "a6b377e3400b08991b80d6805d627f347f983866" + +[[projects]] + branch = "master" + name = "github.com/golang/protobuf" + packages = ["proto"] + revision = "8ee79997227bf9b34611aee7946ae64735e6fd93" + +[[projects]] + branch = "master" + name = "github.com/gorilla/context" + packages = ["."] + revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" + +[[projects]] + name = "github.com/gorilla/mux" + packages = ["."] + revision = "392c28fe23e1c45ddba891b0320b3b5df220beea" + version = "v1.3.0" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/errwrap" + packages = ["."] + revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/go-msgpack" + packages = ["codec"] + revision = "fa3f63826f7c23912c15263591e65d54d080b458" + +[[projects]] + name = "github.com/hashicorp/go-multierror" + packages = ["."] + revision = "ed905158d87462226a13fe39ddf685ea65f1c11f" + +[[projects]] + name = "github.com/hashicorp/hcl" + packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] + revision = "630949a3c5fa3c613328e1b8256052cbc2327c9b" + +[[projects]] + name = "github.com/hashicorp/memberlist" + packages = ["."] + revision = "9800c50ab79c002353852a9b1095e9591b161513" + version = "v0.1.0" + +[[projects]] + branch = "master" + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + +[[projects]] + name = "github.com/magiconair/properties" + packages = ["."] + revision = "b3b15ef068fd0b17ddf408a23669f20811d194d2" + +[[projects]] + name = "github.com/miekg/dns" + packages = ["."] + revision = "ca336a1f95a6b89be9c250df26c7a41742eb4a6f" + +[[projects]] + name = "github.com/mitchellh/mapstructure" + packages = ["."] + revision = "db1efb556f84b25a0a13a04aad883943538ad2e0" + +[[projects]] + name = "github.com/pelletier/go-buffruneio" + packages = ["."] + revision = "c37440a7cf42ac63b919c752ca73a85067e05992" + version = "v0.2.0" + +[[projects]] + name = "github.com/pelletier/go-toml" + packages = ["."] + revision = "23f644976aa7c724adf4aec911dadf4af17840ab" + +[[projects]] + name = "github.com/rakyll/statik" + packages = ["fs"] + revision = "89fe3459b5c829c32e89bdff9c43f18aad728f2f" + +[[projects]] + branch = "master" + name = "github.com/spf13/afero" + packages = [".","mem"] + revision = "9be650865eab0c12963d8753212f4f9c66cdcf12" + +[[projects]] + name = "github.com/spf13/cast" + packages = ["."] + revision = "4f1683a2242a92e62d6ff705a30e435cbf2b50a3" + +[[projects]] + branch = "master" + name = "github.com/spf13/cobra" + packages = ["."] + revision = "fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c" + +[[projects]] + name = "github.com/spf13/jwalterweatherman" + packages = ["."] + revision = "fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "9ff6c6923cfffbcd502984b8e0c80539a94968b7" + +[[projects]] + branch = "master" + name = "github.com/spf13/viper" + packages = ["."] + revision = "7538d73b4eb9511d85a9f1dfef202eeb8ac260f4" + +[[projects]] + name = "golang.org/x/net" + packages = ["context"] + revision = "60c41d1de8da134c05b7b40154a9a82bf5b7edb9" + +[[projects]] + branch = "master" + name = "golang.org/x/sync" + packages = ["errgroup"] + revision = "450f422ab23cf9881c94e2db30cac0eb1b7cf80c" + +[[projects]] + name = "golang.org/x/sys" + packages = ["unix"] + revision = "c200b10b5d5e122be351b67af224adc6128af5bf" + +[[projects]] + name = "golang.org/x/text" + packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"] + revision = "5a42fa2464759cbb7ee0af9de00b54d69f09a29c" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "a3f3340b5840cee44f372bddb5880fcbc419b46a" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "84ff0992f3a6023a9d4832d6aea43d6aec19878a4b9e991ba8ff8269b589e816" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 000000000..b3e45f428 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,49 @@ + +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" + +[[constraint]] + name = "github.com/BurntSushi/toml" + revision = "99064174e013895bbd9b025c31100bd1d9b590ca" + +[[constraint]] + branch = "master" + name = "github.com/CAFxX/gcnotifier" + +[[constraint]] + name = "github.com/boltdb/bolt" + revision = "4b1ebc1869ad66568b313d0dc410e2be72670dda" + +[[constraint]] + name = "github.com/gorilla/mux" + version = "1.3.0" + +[[constraint]] + name = "github.com/hashicorp/memberlist" + version = "=0.1.0" + +[[constraint]] + branch = "master" + name = "github.com/spf13/cobra" + +[[constraint]] + branch = "master" + name = "github.com/spf13/viper" diff --git a/Makefile b/Makefile index d26d6b42c..c3b61d55d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release test cover cover-pkg cover-viz +.PHONY: dep docker pilosa crossbuild install generate statik release test cover cover-pkg cover-viz -GLIDE := $(shell command -v glide 2>/dev/null) +DEP := $(shell command -v dep 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) PROTOC := $(shell command -v protoc 2>/dev/null) VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -15,24 +15,17 @@ default: test pilosa $(GOPATH)/bin: mkdir $(GOPATH)/bin -glide: $(GOPATH)/bin -ifndef GLIDE - curl https://glide.sh/get | sh +dep: $(GOPATH)/bin + go get -u github.com/golang/dep/cmd/dep + +vendor: Gopkg.toml +ifndef DEP + make dep endif + dep ensure -$(GLIDE): - make glide - -vendor: $(GLIDE) glide.yaml -ifndef GLIDE - curl https://glide.sh/get | sh -endif - glide install - -glide.lock: glide glide.yaml - glide update - -vendor-update: glide.lock +Gopkg.lock: dep Gopkg.toml + dep ensure test: vendor go test $(PKGS) $(TESTFLAGS) diff --git a/README-dev.md b/README-dev.md index c5fe72368..809befbc3 100644 --- a/README-dev.md +++ b/README-dev.md @@ -21,12 +21,18 @@ git clone git@github.com:${USER}/pilosa.git cd ${GOPATH}/src/github.com/pilosa/pilosa ``` -Install [Glide][] to manage dependencies. +Install `dep` to manage dependencies: + +```sh +go get -u github.com/golang/dep/cmd/dep +``` Install Pilosa command line tools: ```sh -go install github.com/pilosa/pilosa/cmd/... +make install +# or: +# dep ensure && go install github.com/pilosa/pilosa/cmd/... ``` Running `pilosa` should now run a Pilosa instance. @@ -61,5 +67,3 @@ git push --set-upstream origin a-branch-for-the-task ``` All left to do is creating a pull request on github.com. - -[Glide]: http://glide.sh/ diff --git a/docs/installation.md b/docs/installation.md index 6f1795106..ff24d3c55 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -133,7 +133,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH). * [Git](https://git-scm.com/) - * [Glide](http://glide.sh/) 2. Clone the repo: ``` @@ -286,7 +285,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH). * [Git](https://git-scm.com/) - * [Glide](http://glide.sh/) 2. Clone the repo: ``` diff --git a/glide.lock b/glide.lock deleted file mode 100644 index 45633e798..000000000 --- a/glide.lock +++ /dev/null @@ -1,110 +0,0 @@ -hash: 98b6811c335d30f9711d53472ad3e1ae51090a04751dd6dc3bcb97a54e01afb9 -updated: 2017-05-01T10:42:29.124689561-05:00 -imports: -- name: github.com/armon/go-metrics - version: 97c69685293dce4c0a2d0b19535179bbc976e4d2 -- name: github.com/boltdb/bolt - version: 4b1ebc1869ad66568b313d0dc410e2be72670dda -- name: github.com/BurntSushi/toml - version: 99064174e013895bbd9b025c31100bd1d9b590ca -- name: github.com/CAFxX/gcnotifier - version: adea3e70515666981da25214a7d3e377e4841c22 -- name: github.com/DataDog/datadog-go - version: 909c02b65dd8a52e8fa6072db9752a112227cf21 - subpackages: - - statsd -- name: github.com/davecgh/go-spew - version: 346938d642f2ec3594ed81d874461961cd0faa76 - subpackages: - - spew -- name: github.com/fsnotify/fsnotify - version: 7d7316ed6e1ed2de075aab8dfc76de5d158d66e1 -- name: github.com/gogo/protobuf - version: a9cd0c35b97daf74d0ebf3514c5254814b2703b4 - subpackages: - - proto -- name: github.com/golang/groupcache - version: a6b377e3400b08991b80d6805d627f347f983866 - subpackages: - - lru -- name: github.com/golang/protobuf - version: 8ee79997227bf9b34611aee7946ae64735e6fd93 - subpackages: - - proto -- name: github.com/gorilla/context - version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42 -- name: github.com/gorilla/mux - version: 392c28fe23e1c45ddba891b0320b3b5df220beea -- name: github.com/hashicorp/errwrap - version: 7554cd9344cec97297fa6649b055a8c98c2a1e55 -- name: github.com/hashicorp/go-msgpack - version: fa3f63826f7c23912c15263591e65d54d080b458 - subpackages: - - codec -- name: github.com/hashicorp/go-multierror - version: ed905158d87462226a13fe39ddf685ea65f1c11f -- name: github.com/hashicorp/hcl - version: 630949a3c5fa3c613328e1b8256052cbc2327c9b - subpackages: - - hcl/ast - - hcl/parser - - hcl/scanner - - hcl/strconv - - hcl/token - - json/parser - - json/scanner - - json/token -- name: github.com/hashicorp/memberlist - version: 9800c50ab79c002353852a9b1095e9591b161513 -- name: github.com/inconshreveable/mousetrap - version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 -- name: github.com/magiconair/properties - version: b3b15ef068fd0b17ddf408a23669f20811d194d2 -- name: github.com/miekg/dns - version: ca336a1f95a6b89be9c250df26c7a41742eb4a6f -- name: github.com/mitchellh/mapstructure - version: db1efb556f84b25a0a13a04aad883943538ad2e0 -- name: github.com/pelletier/go-buffruneio - version: c37440a7cf42ac63b919c752ca73a85067e05992 -- name: github.com/pelletier/go-toml - version: 23f644976aa7c724adf4aec911dadf4af17840ab -- name: github.com/rakyll/statik - version: 89fe3459b5c829c32e89bdff9c43f18aad728f2f - subpackages: - - fs -- name: github.com/satori/go.uuid - version: 879c5887cd475cd7864858769793b2ceb0d44feb -- name: github.com/spf13/afero - version: 9be650865eab0c12963d8753212f4f9c66cdcf12 - subpackages: - - mem -- name: github.com/spf13/cast - version: 4f1683a2242a92e62d6ff705a30e435cbf2b50a3 -- name: github.com/spf13/cobra - version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c -- name: github.com/spf13/jwalterweatherman - version: fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66 -- name: github.com/spf13/pflag - version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 -- name: github.com/spf13/viper - version: 7538d73b4eb9511d85a9f1dfef202eeb8ac260f4 -- name: golang.org/x/net - version: 60c41d1de8da134c05b7b40154a9a82bf5b7edb9 - subpackages: - - context -- name: golang.org/x/sync - version: 450f422ab23cf9881c94e2db30cac0eb1b7cf80c - subpackages: - - errgroup -- name: golang.org/x/sys - version: c200b10b5d5e122be351b67af224adc6128af5bf - subpackages: - - unix -- name: golang.org/x/text - version: 5a42fa2464759cbb7ee0af9de00b54d69f09a29c - subpackages: - - transform - - unicode/norm -- name: gopkg.in/yaml.v2 - version: a3f3340b5840cee44f372bddb5880fcbc419b46a -testImports: [] diff --git a/glide.yaml b/glide.yaml deleted file mode 100644 index 32421ffad..000000000 --- a/glide.yaml +++ /dev/null @@ -1,40 +0,0 @@ -package: github.com/pilosa/pilosa -import: -- package: github.com/BurntSushi/toml - version: 99064174e013895bbd9b025c31100bd1d9b590ca -- package: github.com/DataDog/datadog-go - version: ~1.0.0 - subpackages: - - statsd -- package: github.com/boltdb/bolt - version: 4b1ebc1869ad66568b313d0dc410e2be72670dda -- package: github.com/davecgh/go-spew - version: ~1.1.0 - subpackages: - - spew -- package: github.com/gogo/protobuf - version: a9cd0c35b97daf74d0ebf3514c5254814b2703b4 - subpackages: - - proto -- package: github.com/golang/groupcache - version: a6b377e3400b08991b80d6805d627f347f983866 - subpackages: - - lru -- package: golang.org/x/sys - version: c200b10b5d5e122be351b67af224adc6128af5bf - subpackages: - - unix -- package: github.com/golang/protobuf -- package: github.com/satori/go.uuid - version: ^1.1.0 -- package: github.com/spf13/cobra -- package: github.com/spf13/viper -- package: github.com/gorilla/mux - version: ^1.3.0 -- package: github.com/rakyll/statik - version: 89fe3459b5c829c32e89bdff9c43f18aad728f2f - subpackages: - - fs -- package: github.com/hashicorp/memberlist -- package: golang.org/x/sync -- package: github.com/CAFxX/gcnotifier From 4aa487daacfbffaa391a762ec60549e966183c1c Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 21 Jul 2017 13:28:20 -0500 Subject: [PATCH 07/11] check range with CacheTypeNone --- index.go | 3 +-- index_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/index.go b/index.go index a5cdd1911..04903cad5 100644 --- a/index.go +++ b/index.go @@ -394,10 +394,9 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { if opt.RangeEnabled { if opt.InverseEnabled { return nil, ErrInverseRangeNotAllowed - } else if opt.CacheType != "" && opt.CacheType != CacheTypeLRU { + } else if opt.CacheType != "" && opt.CacheType != CacheTypeNone { return nil, ErrRangeCacheNotAllowed } - opt.CacheSize = 0 } else { if len(opt.Fields) > 0 { return nil, ErrFrameFieldsNotAllowed diff --git a/index_test.go b/index_test.go index 2c23f7a40..d3e03ebc7 100644 --- a/index_test.go +++ b/index_test.go @@ -149,6 +149,18 @@ func TestIndex_CreateFrame(t *testing.T) { } }) + t.Run("RangeEnabledWithCacheTypeNone", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + CacheType: pilosa.CacheTypeNone, + CacheSize: uint32(5), + }); err != nil { + t.Fatal(err) + } + }) + t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() From aa73aec6c9bef7e13ad72de823a7e9d9dd77fb09 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 21 Jul 2017 13:31:31 -0500 Subject: [PATCH 08/11] TopN NopCache comment --- fragment_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fragment_test.go b/fragment_test.go index 5a58acd60..982d4a066 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -415,7 +415,7 @@ func TestFragment_TopN_IDs(t *testing.T) { } } -// Ensure a fragment can return top rows when specified by ID. +// Ensure a fragment return none if CacheTypeNone is set func TestFragment_TopN_NopCache(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeNone) defer f.Close() From b6896482b9778973964c37ded7df9189826f767b Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 24 Jul 2017 17:28:23 -0500 Subject: [PATCH 09/11] move InternalPort config option out from under `cluster` and up to the root of `Config` --- config.go | 29 ++++++++++++++++------------- config_test.go | 7 +------ ctl/server.go | 2 +- ctl/server_test.go | 7 ++++--- docs/configuration.md | 23 +++++++++++------------ pilosa.go | 5 ++--- server/server.go | 4 ++-- 7 files changed, 37 insertions(+), 40 deletions(-) diff --git a/config.go b/config.go index 30c1ebdd3..3b248878a 100644 --- a/config.go +++ b/config.go @@ -14,9 +14,7 @@ package pilosa -import ( - "time" -) +import "time" // Cluster types. const ( @@ -51,8 +49,9 @@ var ClusterTypes = []string{ClusterNone, ClusterStatic, ClusterHTTP, ClusterGoss // Config represents the configuration for the command. type Config struct { - DataDir string `toml:"data-dir"` - Bind string `toml:"bind"` + DataDir string `toml:"data-dir"` + Bind string `toml:"bind"` + InternalPort string `toml:"internal-port"` Cluster struct { ReplicaN int `toml:"replicas"` @@ -60,7 +59,6 @@ type Config struct { Hosts []string `toml:"hosts"` InternalHosts []string `toml:"internal-hosts"` PollInterval Duration `toml:"poll-interval"` - InternalPort string `toml:"internal-port"` GossipSeed string `toml:"gossip-seed"` LongQueryTime Duration `toml:"long-query-time"` } `toml:"cluster"` @@ -114,19 +112,24 @@ func (c *Config) Validate() error { if c.Cluster.ReplicaN > len(c.Cluster.Hosts) { return ErrConfigReplicaNInvalid } - if len(c.Cluster.Hosts) != len(c.Cluster.InternalHosts) { - return ErrConfigHostsMismatch - } if !foundItem(c.Cluster.Hosts, c.Bind) { return ErrConfigHostsMissing } - if !ContainsSubstring(c.Cluster.InternalPort, c.Cluster.InternalHosts) { + } + if c.Cluster.Type == ClusterHTTP { + if len(c.Cluster.Hosts) != len(c.Cluster.InternalHosts) { + return ErrConfigHostsMismatch + } + // TODO: this seems like an odd check; it's just ensuring that InternalPort + // matches any one substring from any of the InternalHosts. + // I suggest we either remove this completely or make it actually check + // the port portion of the address for this node. (note that this only applies + // to the http broadcaster, so if we simply use gossip for all implementations + // we can remove this). + if !ContainsSubstring(c.InternalPort, c.Cluster.InternalHosts) { return ErrConfigBroadcastPort } } - if c.Cluster.Type == ClusterGossip && !StringInSlice(c.Cluster.GossipSeed, c.Cluster.InternalHosts) { - return ErrConfigGossipSeed - } return nil } diff --git a/config_test.go b/config_test.go index 78c4a6127..10febfa05 100644 --- a/config_test.go +++ b/config_test.go @@ -26,7 +26,7 @@ func Test_NewConfig(t *testing.T) { t.Fatal(err) } - c.Cluster.InternalPort = pilosa.DefaultInternalPort + c.InternalPort = pilosa.DefaultInternalPort c.Cluster.InternalHosts = []string{"localhost:14004", "localhost:14001"} if err := c.Validate(); err != pilosa.ErrConfigBroadcastPort { t.Fatal(err) @@ -47,11 +47,6 @@ func Test_NewConfig(t *testing.T) { c.Cluster.ReplicaN = 2 c.Cluster.Type = pilosa.ClusterGossip - c.Cluster.GossipSeed = "localhost:10101" - if err := c.Validate(); err != pilosa.ErrConfigGossipSeed { - t.Fatal(err) - } - c.Cluster.GossipSeed = "localhost:14000" if err := c.Validate(); err != nil { t.Fatal(err) diff --git a/ctl/server.go b/ctl/server.go index cde11424b..70f43bc41 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -26,6 +26,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags := cmd.Flags() flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") flags.StringVarP(&srv.Config.Bind, "bind", "b", ":10101", "Default URI on which pilosa should listen.") + flags.StringVarP(&srv.Config.InternalPort, "internal-port", "", "", "Port to which pilosa should bind for internal state sharing.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") @@ -39,7 +40,6 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]") flags.StringVarP(&srv.Config.Cluster.GossipSeed, "cluster.gossip-seed", "", "", "Host with which to seed the gossip membership.") - flags.StringVarP(&srv.Config.Cluster.InternalPort, "cluster.internal-port", "", "", "Port to which pilosa should bind for internal state sharing.") flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.") flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.") flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.") diff --git a/ctl/server_test.go b/ctl/server_test.go index b39c894b1..16dd26916 100644 --- a/ctl/server_test.go +++ b/ctl/server_test.go @@ -16,9 +16,10 @@ package ctl import ( "bytes" + "testing" + "github.com/pilosa/pilosa/server" "github.com/spf13/cobra" - "testing" ) func TestBuildServerFlags(t *testing.T) { @@ -27,8 +28,8 @@ func TestBuildServerFlags(t *testing.T) { stdin, stdout, stderr := GetIO(buf) Server := server.NewCommand(stdin, stdout, stderr) BuildServerFlags(cm, Server) - if cm.Flags().Lookup("cluster.internal-port").Name == "" { - t.Fatal("cluster.internal-port flag is missed ") + if cm.Flags().Lookup("internal-port").Name == "" { + t.Fatal("internal-port flag is missed ") } if cm.Flags().Lookup("data-dir").Name == "" { t.Fatal("data-dir flag is missed ") diff --git a/docs/configuration.md b/docs/configuration.md index 5eecde849..b87d546fe 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -58,6 +58,17 @@ Any flag that has a value that is a comma separated list on the command line bec bind = localhost:10101 ``` +#### Internal Port + +* Description: Port to which Pilosa should bind for internal communication. +* Flag: `--internal-port=11101` +* Env: `PILOSA_INTERNAL_PORT=11101` +* Config: + + ```toml + internal-port = 11101 + ``` + #### Cluster Hosts * Description: List of hosts in the cluster. Multiple hosts should be comma separated in the flag and env forms. @@ -82,18 +93,6 @@ Any flag that has a value that is a comma separated list on the command line bec internal-hosts = ["localhost:11101"] ``` -#### Cluster Internal Port - -* Description: Port to which Pilosa should bind for internal communication. -* Flag: `--cluster.internal-port=11101` -* Env: `PILOSA_CLUSTER.INTERNAL_PORT=11101` -* Config: - - ```toml - [cluster] - internal-port = 11101 - ``` - #### Cluster Poll Interval * Description: Polling interval for cluster. diff --git a/pilosa.go b/pilosa.go index 0b16a05b8..f54c04e06 100644 --- a/pilosa.go +++ b/pilosa.go @@ -65,7 +65,6 @@ var ( ErrConfigBroadcastPort = errors.New("internal-port not found in internal-hosts") ErrConfigHostsMismatch = errors.New("hosts and internal-hosts length mismatch") ErrConfigReplicaNInvalid = errors.New("replica number must be <= hosts") - ErrConfigGossipSeed = errors.New("invalid gossip seed") ) // Regular expression to validate index and frame names. @@ -143,7 +142,7 @@ func ValidateLabel(label string) error { return nil } -// StringInSlice checks is substring a is in the slice +// StringInSlice checks for substring a in the slice. func StringInSlice(a string, list []string) bool { for _, b := range list { if b == a { @@ -153,7 +152,7 @@ func StringInSlice(a string, list []string) bool { return false } -// ContainsSubstring checks is substring a is contained in the slice +// ContainsSubstring checks to see if substring a is contained in any string in the slice. func ContainsSubstring(a string, list []string) bool { for _, b := range list { if strings.Contains(b, a) { diff --git a/server/server.go b/server/server.go index 6e51c4e74..1553917bb 100644 --- a/server/server.go +++ b/server/server.go @@ -153,8 +153,8 @@ func (m *Command) SetupServer() error { // Set internal port (string). internalPortStr := pilosa.DefaultInternalPort - if m.Config.Cluster.InternalPort != "" { - internalPortStr = m.Config.Cluster.InternalPort + if m.Config.InternalPort != "" { + internalPortStr = m.Config.InternalPort } switch m.Config.Cluster.Type { From 326a16209421adb94b99278b3d9cc548296a1e6e Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 24 Jul 2017 22:09:31 -0500 Subject: [PATCH 10/11] fixed review --- cache.go | 8 ++++---- fragment.go | 2 +- index.go | 3 --- index_test.go | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/cache.go b/cache.go index d6d4174cd..242e5a6ab 100644 --- a/cache.go +++ b/cache.go @@ -484,19 +484,19 @@ func (s *SimpleCache) Add(id uint64, b *Bitmap) { s.cache[id] = b } +// NopCache represents a no-op Cache implementation. type NopCache struct { stats StatsClient } -// NopCache implement Cache interface, returns no cache for cache type None +// Ensure NopCache implements Cache. var _ Cache = &NopCache{} -// NewNopeCache returns a new instance of NopCache. +// NewNopCache returns a new instance of NopCache. func NewNopCache() *NopCache { - c := &NopCache{ + return &NopCache{ stats: NopStatsClient, } - return c } func (c *NopCache) Add(id uint64, n uint64) {} diff --git a/fragment.go b/fragment.go index 16ec5869f..b20dfd9c0 100644 --- a/fragment.go +++ b/fragment.go @@ -704,7 +704,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { } func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair { - // Don't retrieve from storage if CacheTypeNone + // Don't retrieve from storage if CacheTypeNone. if f.CacheType == CacheTypeNone { return f.cache.Top() } diff --git a/index.go b/index.go index 04903cad5..a07d28dfe 100644 --- a/index.go +++ b/index.go @@ -388,9 +388,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { } // Validate mutually exclusive options if ranges are enabled. - // - // NOTE(https://github.com/pilosa/pilosa/issues/399): - // Cache type should be validated as "none" once it is allowed. if opt.RangeEnabled { if opt.InverseEnabled { return nil, ErrInverseRangeNotAllowed diff --git a/index_test.go b/index_test.go index d3e03ebc7..0d17a72af 100644 --- a/index_test.go +++ b/index_test.go @@ -155,7 +155,7 @@ func TestIndex_CreateFrame(t *testing.T) { if _, err := index.CreateFrame("f", pilosa.FrameOptions{ RangeEnabled: true, CacheType: pilosa.CacheTypeNone, - CacheSize: uint32(5), + CacheSize: uint32(5), }); err != nil { t.Fatal(err) } From 17c6dc3ae43e6f5fbbb19d203bf25a65338769b5 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 25 Jul 2017 16:55:38 -0500 Subject: [PATCH 11/11] Run `touch vendor` in `make vendor` so that `dep ensure` is only run when needed. --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index c3b61d55d..b064c7ce0 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ ifndef DEP make dep endif dep ensure + touch vendor Gopkg.lock: dep Gopkg.toml dep ensure