scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into input-definition

Browse source
This commit is contained in:
Travis 2017-07-26 14:44:31 -05:00
commit b91da2d0a4
No known key found for this signature in database
GPG key ID: 7F08008DFD9314C9
24 changed files with 416 additions and 235 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.travis.yml
View file

@ -4,11 +4,6 @@ go:
- 1.8
- master
addons:
apt:
sources:
- sourceline: 'ppa:masterminds/glide'
packages:
- glide
before_install:
- go get github.com/mattn/goveralls
script:

10
Dockerfile
View file

@ -1,16 +1,10 @@
FROM golang:1.8.1 as builder
FROM golang:1.8.3 as builder
ARG ldflags=''
ARG GLIDE="https://github.com/Masterminds/glide/releases/download/v0.12.3/glide-v0.12.3-linux-amd64.tar.gz"
ARG GLIDE_HASH="d6d3816c70fba716466e7381a9c06cb31565a3b87acb5bad9dd3beb0a9f9b0f8"
COPY . /go/src/github.com/pilosa/pilosa
RUN wget ${GLIDE} -O /go/glide.tar.gz -q \
&& tar xf /go/glide.tar.gz \
&& mv /go/linux-amd64/glide /go/bin \
&& echo "${GLIDE_HASH} /go/bin/glide" | shasum -a 256 -c - \
&& cd /go/src/github.com/pilosa/pilosa \
RUN cd /go/src/github.com/pilosa/pilosa \
&& make vendor \
&& CGO_ENABLED=0 go install -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa

199
Gopkg.lock generated Normal file
View file

@ -0,0 +1,199 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "github.com/BurntSushi/toml"
packages = ["."]
revision = "99064174e013895bbd9b025c31100bd1d9b590ca"
[[projects]]
branch = "master"
name = "github.com/CAFxX/gcnotifier"
packages = ["."]
revision = "adea3e70515666981da25214a7d3e377e4841c22"
[[projects]]
name = "github.com/DataDog/datadog-go"
packages = ["statsd"]
revision = "909c02b65dd8a52e8fa6072db9752a112227cf21"
version = "1.0.0"
[[projects]]
name = "github.com/armon/go-metrics"
packages = ["."]
revision = "97c69685293dce4c0a2d0b19535179bbc976e4d2"
[[projects]]
name = "github.com/boltdb/bolt"
packages = ["."]
revision = "4b1ebc1869ad66568b313d0dc410e2be72670dda"
[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
name = "github.com/fsnotify/fsnotify"
packages = ["."]
revision = "7d7316ed6e1ed2de075aab8dfc76de5d158d66e1"
[[projects]]
name = "github.com/gogo/protobuf"
packages = ["proto"]
revision = "a9cd0c35b97daf74d0ebf3514c5254814b2703b4"
[[projects]]
name = "github.com/golang/groupcache"
packages = ["lru"]
revision = "a6b377e3400b08991b80d6805d627f347f983866"
[[projects]]
branch = "master"
name = "github.com/golang/protobuf"
packages = ["proto"]
revision = "8ee79997227bf9b34611aee7946ae64735e6fd93"
[[projects]]
branch = "master"
name = "github.com/gorilla/context"
packages = ["."]
revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42"
[[projects]]
name = "github.com/gorilla/mux"
packages = ["."]
revision = "392c28fe23e1c45ddba891b0320b3b5df220beea"
version = "v1.3.0"
[[projects]]
branch = "master"
name = "github.com/hashicorp/errwrap"
packages = ["."]
revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
[[projects]]
branch = "master"
name = "github.com/hashicorp/go-msgpack"
packages = ["codec"]
revision = "fa3f63826f7c23912c15263591e65d54d080b458"
[[projects]]
name = "github.com/hashicorp/go-multierror"
packages = ["."]
revision = "ed905158d87462226a13fe39ddf685ea65f1c11f"
[[projects]]
name = "github.com/hashicorp/hcl"
packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"]
revision = "630949a3c5fa3c613328e1b8256052cbc2327c9b"
[[projects]]
name = "github.com/hashicorp/memberlist"
packages = ["."]
revision = "9800c50ab79c002353852a9b1095e9591b161513"
version = "v0.1.0"
[[projects]]
branch = "master"
name = "github.com/inconshreveable/mousetrap"
packages = ["."]
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
[[projects]]
name = "github.com/magiconair/properties"
packages = ["."]
revision = "b3b15ef068fd0b17ddf408a23669f20811d194d2"
[[projects]]
name = "github.com/miekg/dns"
packages = ["."]
revision = "ca336a1f95a6b89be9c250df26c7a41742eb4a6f"
[[projects]]
name = "github.com/mitchellh/mapstructure"
packages = ["."]
revision = "db1efb556f84b25a0a13a04aad883943538ad2e0"
[[projects]]
name = "github.com/pelletier/go-buffruneio"
packages = ["."]
revision = "c37440a7cf42ac63b919c752ca73a85067e05992"
version = "v0.2.0"
[[projects]]
name = "github.com/pelletier/go-toml"
packages = ["."]
revision = "23f644976aa7c724adf4aec911dadf4af17840ab"
[[projects]]
name = "github.com/rakyll/statik"
packages = ["fs"]
revision = "89fe3459b5c829c32e89bdff9c43f18aad728f2f"
[[projects]]
branch = "master"
name = "github.com/spf13/afero"
packages = [".","mem"]
revision = "9be650865eab0c12963d8753212f4f9c66cdcf12"
[[projects]]
name = "github.com/spf13/cast"
packages = ["."]
revision = "4f1683a2242a92e62d6ff705a30e435cbf2b50a3"
[[projects]]
branch = "master"
name = "github.com/spf13/cobra"
packages = ["."]
revision = "fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c"
[[projects]]
name = "github.com/spf13/jwalterweatherman"
packages = ["."]
revision = "fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66"
[[projects]]
name = "github.com/spf13/pflag"
packages = ["."]
revision = "9ff6c6923cfffbcd502984b8e0c80539a94968b7"
[[projects]]
branch = "master"
name = "github.com/spf13/viper"
packages = ["."]
revision = "7538d73b4eb9511d85a9f1dfef202eeb8ac260f4"
[[projects]]
name = "golang.org/x/net"
packages = ["context"]
revision = "60c41d1de8da134c05b7b40154a9a82bf5b7edb9"
[[projects]]
branch = "master"
name = "golang.org/x/sync"
packages = ["errgroup"]
revision = "450f422ab23cf9881c94e2db30cac0eb1b7cf80c"
[[projects]]
name = "golang.org/x/sys"
packages = ["unix"]
revision = "c200b10b5d5e122be351b67af224adc6128af5bf"
[[projects]]
name = "golang.org/x/text"
packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"]
revision = "5a42fa2464759cbb7ee0af9de00b54d69f09a29c"
[[projects]]
name = "gopkg.in/yaml.v2"
packages = ["."]
revision = "a3f3340b5840cee44f372bddb5880fcbc419b46a"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "84ff0992f3a6023a9d4832d6aea43d6aec19878a4b9e991ba8ff8269b589e816"
solver-name = "gps-cdcl"
solver-version = 1

49
Gopkg.toml Normal file
View file

@ -0,0 +1,49 @@
# Gopkg.toml example
#
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
[[constraint]]
name = "github.com/BurntSushi/toml"
revision = "99064174e013895bbd9b025c31100bd1d9b590ca"
[[constraint]]
branch = "master"
name = "github.com/CAFxX/gcnotifier"
[[constraint]]
name = "github.com/boltdb/bolt"
revision = "4b1ebc1869ad66568b313d0dc410e2be72670dda"
[[constraint]]
name = "github.com/gorilla/mux"
version = "1.3.0"
[[constraint]]
name = "github.com/hashicorp/memberlist"
version = "=0.1.0"
[[constraint]]
branch = "master"
name = "github.com/spf13/cobra"
[[constraint]]
branch = "master"
name = "github.com/spf13/viper"

30
Makefile
View file

@ -1,6 +1,6 @@
.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release test cover cover-pkg cover-viz
.PHONY: dep docker pilosa crossbuild install generate statik release test cover cover-pkg cover-viz
GLIDE := $(shell command -v glide 2>/dev/null)
DEP := $(shell command -v dep 2>/dev/null)
STATIK := $(shell command -v statik 2>/dev/null)
PROTOC := $(shell command -v protoc 2>/dev/null)
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@ -15,24 +15,18 @@ default: test pilosa
$(GOPATH)/bin:
mkdir $(GOPATH)/bin
glide: $(GOPATH)/bin
ifndef GLIDE
curl https://glide.sh/get | sh
dep: $(GOPATH)/bin
go get -u github.com/golang/dep/cmd/dep
vendor: Gopkg.toml
ifndef DEP
make dep
endif
dep ensure
touch vendor
$(GLIDE):
make glide
vendor: $(GLIDE) glide.yaml
ifndef GLIDE
curl https://glide.sh/get | sh
endif
glide install
glide.lock: glide glide.yaml
glide update
vendor-update: glide.lock
Gopkg.lock: dep Gopkg.toml
dep ensure
test: vendor
go test $(PKGS) $(TESTFLAGS)

12
README-dev.md
View file

@ -21,12 +21,18 @@ git clone git@github.com:${USER}/pilosa.git
cd ${GOPATH}/src/github.com/pilosa/pilosa
```
Install [Glide][] to manage dependencies.
Install `dep` to manage dependencies:
```sh
go get -u github.com/golang/dep/cmd/dep
```
Install Pilosa command line tools:
```sh
go install github.com/pilosa/pilosa/cmd/...
make install
# or:
# dep ensure && go install github.com/pilosa/pilosa/cmd/...
```
Running `pilosa` should now run a Pilosa instance.
@ -61,5 +67,3 @@ git push --set-upstream origin a-branch-for-the-task
```
All left to do is creating a pull request on github.com.
[Glide]: http://glide.sh/

34
cache.go
View file

@ -54,7 +54,7 @@ type Cache interface {
SetStats(s StatsClient)
}
// LRUCache represents a least recently used Cache implemenation.
// LRUCache represents a least recently used Cache implementation.
type LRUCache struct {
cache *lru.Cache
counts map[uint64]uint64
@ -483,3 +483,35 @@ func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) {
func (s *SimpleCache) Add(id uint64, b *Bitmap) {
s.cache[id] = b
}
// NopCache represents a no-op Cache implementation.
type NopCache struct {
stats StatsClient
}
// Ensure NopCache implements Cache.
var _ Cache = &NopCache{}
// NewNopCache returns a new instance of NopCache.
func NewNopCache() *NopCache {
return &NopCache{
stats: NopStatsClient,
}
}
func (c *NopCache) Add(id uint64, n uint64) {}
func (c *NopCache) BulkAdd(id uint64, n uint64) {}
func (c *NopCache) Get(id uint64) uint64 { return 0 }
func (c *NopCache) IDs() []uint64 { return make([]uint64, 0, 0) }
func (c *NopCache) Invalidate() {}
func (c *NopCache) Len() int { return 0 }
func (c *NopCache) Recalculate() {
}
func (c *NopCache) SetStats(s StatsClient) {
c.stats = s
}
func (c *NopCache) Top() []BitmapPair {
return []BitmapPair{}
}

29
config.go
View file

@ -14,9 +14,7 @@
package pilosa
import (
"time"
)
import "time"
// Cluster types.
const (
@ -51,8 +49,9 @@ var ClusterTypes = []string{ClusterNone, ClusterStatic, ClusterHTTP, ClusterGoss
// Config represents the configuration for the command.
type Config struct {
DataDir string `toml:"data-dir"`
Bind string `toml:"bind"`
DataDir string `toml:"data-dir"`
Bind string `toml:"bind"`
InternalPort string `toml:"internal-port"`
Cluster struct {
ReplicaN int `toml:"replicas"`
@ -60,7 +59,6 @@ type Config struct {
Hosts []string `toml:"hosts"`
InternalHosts []string `toml:"internal-hosts"`
PollInterval Duration `toml:"poll-interval"`
InternalPort string `toml:"internal-port"`
GossipSeed string `toml:"gossip-seed"`
LongQueryTime Duration `toml:"long-query-time"`
} `toml:"cluster"`
@ -114,19 +112,24 @@ func (c *Config) Validate() error {
if c.Cluster.ReplicaN > len(c.Cluster.Hosts) {
return ErrConfigReplicaNInvalid
}
if len(c.Cluster.Hosts) != len(c.Cluster.InternalHosts) {
return ErrConfigHostsMismatch
}
if !foundItem(c.Cluster.Hosts, c.Bind) {
return ErrConfigHostsMissing
}
if !ContainsSubstring(c.Cluster.InternalPort, c.Cluster.InternalHosts) {
}
if c.Cluster.Type == ClusterHTTP {
if len(c.Cluster.Hosts) != len(c.Cluster.InternalHosts) {
return ErrConfigHostsMismatch
}
// TODO: this seems like an odd check; it's just ensuring that InternalPort
// matches any one substring from any of the InternalHosts.
// I suggest we either remove this completely or make it actually check
// the port portion of the address for this node. (note that this only applies
// to the http broadcaster, so if we simply use gossip for all implementations
// we can remove this).
if !ContainsSubstring(c.InternalPort, c.Cluster.InternalHosts) {
return ErrConfigBroadcastPort
}
}
if c.Cluster.Type == ClusterGossip && !StringInSlice(c.Cluster.GossipSeed, c.Cluster.InternalHosts) {
return ErrConfigGossipSeed
}
return nil
}

7
config_test.go
View file

@ -26,7 +26,7 @@ func Test_NewConfig(t *testing.T) {
t.Fatal(err)
}
c.Cluster.InternalPort = pilosa.DefaultInternalPort
c.InternalPort = pilosa.DefaultInternalPort
c.Cluster.InternalHosts = []string{"localhost:14004", "localhost:14001"}
if err := c.Validate(); err != pilosa.ErrConfigBroadcastPort {
t.Fatal(err)
@ -47,11 +47,6 @@ func Test_NewConfig(t *testing.T) {
c.Cluster.ReplicaN = 2
c.Cluster.Type = pilosa.ClusterGossip
c.Cluster.GossipSeed = "localhost:10101"
if err := c.Validate(); err != pilosa.ErrConfigGossipSeed {
t.Fatal(err)
}
c.Cluster.GossipSeed = "localhost:14000"
if err := c.Validate(); err != nil {
t.Fatal(err)

2
ctl/server.go
View file

@ -26,6 +26,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags := cmd.Flags()
flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.")
flags.StringVarP(&srv.Config.Bind, "bind", "b", ":10101", "Default URI on which pilosa should listen.")
flags.StringVarP(&srv.Config.InternalPort, "internal-port", "", "", "Port to which pilosa should bind for internal state sharing.")
flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.")
@ -39,7 +40,6 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.")
flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]")
flags.StringVarP(&srv.Config.Cluster.GossipSeed, "cluster.gossip-seed", "", "", "Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Cluster.InternalPort, "cluster.internal-port", "", "", "Port to which pilosa should bind for internal state sharing.")
flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.")
flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.")
flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.")

7
ctl/server_test.go
View file

@ -16,9 +16,10 @@ package ctl
import (
"bytes"
"testing"
"github.com/pilosa/pilosa/server"
"github.com/spf13/cobra"
"testing"
)
func TestBuildServerFlags(t *testing.T) {
@ -27,8 +28,8 @@ func TestBuildServerFlags(t *testing.T) {
stdin, stdout, stderr := GetIO(buf)
Server := server.NewCommand(stdin, stdout, stderr)
BuildServerFlags(cm, Server)
if cm.Flags().Lookup("cluster.internal-port").Name == "" {
t.Fatal("cluster.internal-port flag is missed ")
if cm.Flags().Lookup("internal-port").Name == "" {
t.Fatal("internal-port flag is missed ")
}
if cm.Flags().Lookup("data-dir").Name == "" {
t.Fatal("data-dir flag is missed ")

23
docs/configuration.md
View file

@ -58,6 +58,17 @@ Any flag that has a value that is a comma separated list on the command line bec
bind = localhost:10101
```
#### Internal Port
* Description: Port to which Pilosa should bind for internal communication.
* Flag: `--internal-port=11101`
* Env: `PILOSA_INTERNAL_PORT=11101`
* Config:
```toml
internal-port = 11101
```
#### Cluster Hosts
* Description: List of hosts in the cluster. Multiple hosts should be comma separated in the flag and env forms.
@ -82,18 +93,6 @@ Any flag that has a value that is a comma separated list on the command line bec
internal-hosts = ["localhost:11101"]
```
#### Cluster Internal Port
* Description: Port to which Pilosa should bind for internal communication.
* Flag: `--cluster.internal-port=11101`
* Env: `PILOSA_CLUSTER.INTERNAL_PORT=11101`
* Config:
```toml
[cluster]
internal-port = 11101
```
#### Cluster Poll Interval
* Description: Polling interval for cluster.

5
docs/installation.md
View file

@ -133,16 +133,16 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH).
* [Git](https://git-scm.com/)
* [Glide](http://glide.sh/)
2. Clone the repo:
```
go get -d github.com/pilosa/pilosa
```
3. Build the Pilosa repo:
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice web console into Pilosa):
```
cd $GOPATH/src/github.com/pilosa/pilosa
make generate-statik
make install
```
@ -286,7 +286,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH).
* [Git](https://git-scm.com/)
* [Glide](http://glide.sh/)
2. Clone the repo:
```

6
fragment.go
View file

@ -249,6 +249,8 @@ func (f *Fragment) openCache() error {
f.cache = NewRankCache(f.CacheSize)
case CacheTypeLRU:
f.cache = NewLRUCache(f.CacheSize)
case CacheTypeNone:
f.cache = NewNopCache()
default:
return ErrInvalidCacheType
}
@ -702,6 +704,10 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
}
func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair {
// Don't retrieve from storage if CacheTypeNone.
if f.CacheType == CacheTypeNone {
return f.cache.Top()
}
// If no specific rows are requested, retrieve top rows.
if len(rowIDs) == 0 {
f.mu.Lock()

18
fragment_test.go
View file

@ -416,6 +416,24 @@ func TestFragment_TopN_IDs(t *testing.T) {
}
}
// Ensure a fragment return none if CacheTypeNone is set
func TestFragment_TopN_NopCache(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeNone)
defer f.Close()
// Set bits on various rows.
f.MustSetBits(100, 1, 2, 3)
f.MustSetBits(101, 4, 5, 6, 7)
f.MustSetBits(102, 8, 9, 10, 11, 12)
// Retrieve top rows.
if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{}) {
t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs))
}
}
// Ensure the fragment cache limit works
func TestFragment_TopN_CacheSize(t *testing.T) {
slice := uint64(0)

3
frame.go
View file

@ -903,12 +903,13 @@ func (p importBitSet) Less(i, j int) bool { return p.rowIDs[i] < p.rowIDs[j] }
const (
CacheTypeLRU = "lru"
CacheTypeRanked = "ranked"
CacheTypeNone = "none"
)
// IsValidCacheType returns true if v is a valid cache type.
func IsValidCacheType(v string) bool {
switch v {
case CacheTypeLRU, CacheTypeRanked:
case CacheTypeLRU, CacheTypeRanked, CacheTypeNone:
return true
default:
return false

110
glide.lock generated
View file

@ -1,110 +0,0 @@
hash: 98b6811c335d30f9711d53472ad3e1ae51090a04751dd6dc3bcb97a54e01afb9
updated: 2017-05-01T10:42:29.124689561-05:00
imports:
- name: github.com/armon/go-metrics
version: 97c69685293dce4c0a2d0b19535179bbc976e4d2
- name: github.com/boltdb/bolt
version: 4b1ebc1869ad66568b313d0dc410e2be72670dda
- name: github.com/BurntSushi/toml
version: 99064174e013895bbd9b025c31100bd1d9b590ca
- name: github.com/CAFxX/gcnotifier
version: adea3e70515666981da25214a7d3e377e4841c22
- name: github.com/DataDog/datadog-go
version: 909c02b65dd8a52e8fa6072db9752a112227cf21
subpackages:
- statsd
- name: github.com/davecgh/go-spew
version: 346938d642f2ec3594ed81d874461961cd0faa76
subpackages:
- spew
- name: github.com/fsnotify/fsnotify
version: 7d7316ed6e1ed2de075aab8dfc76de5d158d66e1
- name: github.com/gogo/protobuf
version: a9cd0c35b97daf74d0ebf3514c5254814b2703b4
subpackages:
- proto
- name: github.com/golang/groupcache
version: a6b377e3400b08991b80d6805d627f347f983866
subpackages:
- lru
- name: github.com/golang/protobuf
version: 8ee79997227bf9b34611aee7946ae64735e6fd93
subpackages:
- proto
- name: github.com/gorilla/context
version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42
- name: github.com/gorilla/mux
version: 392c28fe23e1c45ddba891b0320b3b5df220beea
- name: github.com/hashicorp/errwrap
version: 7554cd9344cec97297fa6649b055a8c98c2a1e55
- name: github.com/hashicorp/go-msgpack
version: fa3f63826f7c23912c15263591e65d54d080b458
subpackages:
- codec
- name: github.com/hashicorp/go-multierror
version: ed905158d87462226a13fe39ddf685ea65f1c11f
- name: github.com/hashicorp/hcl
version: 630949a3c5fa3c613328e1b8256052cbc2327c9b
subpackages:
- hcl/ast
- hcl/parser
- hcl/scanner
- hcl/strconv
- hcl/token
- json/parser
- json/scanner
- json/token
- name: github.com/hashicorp/memberlist
version: 9800c50ab79c002353852a9b1095e9591b161513
- name: github.com/inconshreveable/mousetrap
version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
- name: github.com/magiconair/properties
version: b3b15ef068fd0b17ddf408a23669f20811d194d2
- name: github.com/miekg/dns
version: ca336a1f95a6b89be9c250df26c7a41742eb4a6f
- name: github.com/mitchellh/mapstructure
version: db1efb556f84b25a0a13a04aad883943538ad2e0
- name: github.com/pelletier/go-buffruneio
version: c37440a7cf42ac63b919c752ca73a85067e05992
- name: github.com/pelletier/go-toml
version: 23f644976aa7c724adf4aec911dadf4af17840ab
- name: github.com/rakyll/statik
version: 89fe3459b5c829c32e89bdff9c43f18aad728f2f
subpackages:
- fs
- name: github.com/satori/go.uuid
version: 879c5887cd475cd7864858769793b2ceb0d44feb
- name: github.com/spf13/afero
version: 9be650865eab0c12963d8753212f4f9c66cdcf12
subpackages:
- mem
- name: github.com/spf13/cast
version: 4f1683a2242a92e62d6ff705a30e435cbf2b50a3
- name: github.com/spf13/cobra
version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c
- name: github.com/spf13/jwalterweatherman
version: fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66
- name: github.com/spf13/pflag
version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7
- name: github.com/spf13/viper
version: 7538d73b4eb9511d85a9f1dfef202eeb8ac260f4
- name: golang.org/x/net
version: 60c41d1de8da134c05b7b40154a9a82bf5b7edb9
subpackages:
- context
- name: golang.org/x/sync
version: 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
subpackages:
- errgroup
- name: golang.org/x/sys
version: c200b10b5d5e122be351b67af224adc6128af5bf
subpackages:
- unix
- name: golang.org/x/text
version: 5a42fa2464759cbb7ee0af9de00b54d69f09a29c
subpackages:
- transform
- unicode/norm
- name: gopkg.in/yaml.v2
version: a3f3340b5840cee44f372bddb5880fcbc419b46a
testImports: []

40
glide.yaml
View file

@ -1,40 +0,0 @@
package: github.com/pilosa/pilosa
import:
- package: github.com/BurntSushi/toml
version: 99064174e013895bbd9b025c31100bd1d9b590ca
- package: github.com/DataDog/datadog-go
version: ~1.0.0
subpackages:
- statsd
- package: github.com/boltdb/bolt
version: 4b1ebc1869ad66568b313d0dc410e2be72670dda
- package: github.com/davecgh/go-spew
version: ~1.1.0
subpackages:
- spew
- package: github.com/gogo/protobuf
version: a9cd0c35b97daf74d0ebf3514c5254814b2703b4
subpackages:
- proto
- package: github.com/golang/groupcache
version: a6b377e3400b08991b80d6805d627f347f983866
subpackages:
- lru
- package: golang.org/x/sys
version: c200b10b5d5e122be351b67af224adc6128af5bf
subpackages:
- unix
- package: github.com/golang/protobuf
- package: github.com/satori/go.uuid
version: ^1.1.0
- package: github.com/spf13/cobra
- package: github.com/spf13/viper
- package: github.com/gorilla/mux
version: ^1.3.0
- package: github.com/rakyll/statik
version: 89fe3459b5c829c32e89bdff9c43f18aad728f2f
subpackages:
- fs
- package: github.com/hashicorp/memberlist
- package: golang.org/x/sync
- package: github.com/CAFxX/gcnotifier

11
handler.go
View file

@ -29,6 +29,7 @@ import (
"net/http"
_ "net/http/pprof"
"os"
"runtime/debug"
"strconv"
"strings"
"time"
@ -142,6 +143,16 @@ func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request
// ServeHTTP handles an HTTP request.
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
defer func() {
if err := recover(); err != nil {
w.WriteHeader(http.StatusInternalServerError)
stack := debug.Stack()
msg := "PANIC: %s\n%s"
fmt.Fprintf(h.LogOutput, msg, err, stack)
fmt.Fprintf(w, msg, err, stack)
}
}()
t := time.Now()
h.Router.ServeHTTP(w, r)
dif := time.Since(t)

24
handler_test.go
View file

@ -33,6 +33,30 @@ import (
"github.com/pilosa/pilosa/test"
)
func TestHandlerPanics(t *testing.T) {
h := test.NewHandler()
buf := &bytes.Buffer{}
h.Handler.LogOutput = buf
w := httptest.NewRecorder()
// will panic since Handler has no Holder set up
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil))
bufbytes, err := ioutil.ReadAll(buf)
if err != nil {
t.Fatalf("reading all logoutput: %v", err)
}
if !bytes.Contains(bufbytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) {
t.Fatalf("expected panic in log, but got: %s", bufbytes)
}
if w.Code != http.StatusInternalServerError {
t.Fatalf("expected internal server error, but got: %v", w.Code)
}
bodyBytes := w.Body.Bytes()
if !bytes.Contains(bodyBytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) {
t.Fatalf("response to client should have panic, but got %s", bodyBytes)
}
}
// Ensure the handler returns "not found" for invalid paths.
func TestHandler_NotFound(t *testing.T) {
hldr := test.MustOpenHolder()

6
index.go
View file

@ -414,16 +414,12 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
}
// Validate mutually exclusive options if ranges are enabled.
//
// NOTE(https://github.com/pilosa/pilosa/issues/399):
// Cache type should be validated as "none" once it is allowed.
if opt.RangeEnabled {
if opt.InverseEnabled {
return nil, ErrInverseRangeNotAllowed
} else if opt.CacheType != "" && opt.CacheType != CacheTypeLRU {
} else if opt.CacheType != "" && opt.CacheType != CacheTypeNone {
return nil, ErrRangeCacheNotAllowed
}
opt.CacheSize = 0
} else {
if len(opt.Fields) > 0 {
return nil, ErrFrameFieldsNotAllowed

12
index_test.go
View file

@ -152,6 +152,18 @@ func TestIndex_CreateFrame(t *testing.T) {
}
})
t.Run("RangeEnabledWithCacheTypeNone", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
CacheType: pilosa.CacheTypeNone,
CacheSize: uint32(5),
}); err != nil {
t.Fatal(err)
}
})
t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()

5
pilosa.go
View file

@ -75,7 +75,6 @@ var (
ErrConfigBroadcastPort = errors.New("internal-port not found in internal-hosts")
ErrConfigHostsMismatch = errors.New("hosts and internal-hosts length mismatch")
ErrConfigReplicaNInvalid = errors.New("replica number must be <= hosts")
ErrConfigGossipSeed = errors.New("invalid gossip seed")
)
// Regular expression to validate index and frame names.
@ -153,7 +152,7 @@ func ValidateLabel(label string) error {
return nil
}
// StringInSlice checks is substring a is in the slice
// StringInSlice checks for substring a in the slice.
func StringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
@ -163,7 +162,7 @@ func StringInSlice(a string, list []string) bool {
return false
}
// ContainsSubstring checks is substring a is contained in the slice
// ContainsSubstring checks to see if substring a is contained in any string in the slice.
func ContainsSubstring(a string, list []string) bool {
for _, b := range list {
if strings.Contains(b, a) {

4
server/server.go
View file

@ -153,8 +153,8 @@ func (m *Command) SetupServer() error {
// Set internal port (string).
internalPortStr := pilosa.DefaultInternalPort
if m.Config.Cluster.InternalPort != "" {
internalPortStr = m.Config.Cluster.InternalPort
if m.Config.InternalPort != "" {
internalPortStr = m.Config.InternalPort
}
switch m.Config.Cluster.Type {