diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..081a20073 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,24 @@ +## Overview + +[Describe what this pull request addresses.] + +Fixes # + +## Pull request checklist + +- [ ] I have read the [contributing guide](https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md). +- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa). +- [ ] I have updated the [documentation](https://github.com/pilosa/pilosa/tree/master/docs). +- [ ] I have resolved any merge conflicts. +- [ ] I have included tests that cover my changes. +- [ ] All new and existing tests pass. + +## Code review checklist +This is the checklist that the reviewer will follow while reviewing your pull request. You do not need to do anything with this checklist, but be aware of what the reviewer will be looking for. + +- [ ] Ensure that any changes to external docs have been included in this pull request. +- [ ] If the changes require that minor/major versions need to be updated, tag the PR appropriately. +- [ ] Ensure the new code is [properly commented](https://github.com/golang/go/wiki/CodeReviewComments#doc-comments) and follows [Idiomatic Go](https://dmitri.shuralyov.com/idiomatic-go). +- [ ] Check that tests have been written and that they cover the new functionality. +- [ ] Run tests and ensure they pass. +- [ ] Build and run the code, performing any applicable integration testing. diff --git a/Makefile b/Makefile index 563059137..6ef21fda0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release test +.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release test cover cover-pkg cover-viz GLIDE := $(shell command -v glide 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) @@ -6,8 +6,9 @@ PROTOC := $(shell command -v protoc 2>/dev/null) VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa +PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) BUILD_TIME=`date -u +%FT%T%z` -LDFLAGS=-"-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" +LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" default: test pilosa @@ -23,6 +24,9 @@ $(GLIDE): make glide vendor: $(GLIDE) glide.yaml +ifndef GLIDE + curl https://glide.sh/get | sh +endif glide install glide.lock: glide glide.yaml @@ -31,7 +35,23 @@ glide.lock: glide glide.yaml vendor-update: glide.lock test: vendor - go test $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) $(TESTFLAGS) + go test $(PKGS) $(TESTFLAGS) + +cover: vendor + mkdir -p build/coverage + echo "mode: set" > build/coverage/all.out + for pkg in $(PKGS) ; do \ + make cover-pkg PKG=$$pkg ; \ + done + +cover-pkg: + mkdir -p build/coverage + touch build/coverage/$(subst /,-,$(PKG)).out + go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG) + tail +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out + +cover-viz: cover + go tool cover -html=build/coverage/all.out pilosa: vendor go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa diff --git a/docs/administration.md b/docs/administration.md index 322b68e12..e562f94ce 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -157,3 +157,5 @@ We currently track the following events Garbage Collection: Event count when Garbage Collection occurs. Goroutines: Number of running Goroutines. + +OpenFiles: Number of open file handles associated with running Pilosa process ID. \ No newline at end of file diff --git a/docs/api-reference.md b/docs/api-reference.md index 55bbde24e..39f66683f 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -240,6 +240,6 @@ curl -XGET localhost:10101/version Response: ``` -{"version":"v0.3.0-353-ge633247"} +{"version":"v0.4.0"} ``` diff --git a/docs/installation.md b/docs/installation.md index af04cc240..6f1795106 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -9,25 +9,79 @@ Pilosa is currently available for [MacOS](#installing-on-macos) and [Linux](#ins ### Installing on MacOS -There are three ways to install Pilosa on MacOS: download the binary (recommended), build from source, or use Docker. +There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) (recommended), download the binary, build from source, or use Docker. + +#### Use Homebrew + +1. Update your Homebrew formulas: + ``` + brew update + ``` + +2. Install Pilosa + ``` + brew install pilosa + ``` + +3. Make sure Pilosa is installed successfully: + ``` + pilosa + ``` + + If you see something like: + ``` + Pilosa is a fast index to turbocharge your database. + + This binary contains Pilosa itself, as well as common + tools for administering pilosa, importing/exporting data, + backing up, and more. Complete documentation is available + at https://www.pilosa.com/docs/ + + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 + + Usage: + pilosa [command] + + Available Commands: + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. + + Flags: + -c, --config string Configuration file to read from. + + Use "pilosa [command] --help" for more information about a command. + ``` + + You're good to go! #### Download the Binary 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.3.1/pilosa-v0.3.1-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.4.0/pilosa-v0.4.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.3.1-darwin-amd64.tar.gz + tar xfz pilosa-v0.4.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.3.1-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.4.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -42,25 +96,28 @@ There are three ways to install Pilosa on MacOS: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. + + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. Flags: -c, --config string Configuration file to read from. @@ -101,25 +158,28 @@ There are three ways to install Pilosa on MacOS: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. + Flags: -c, --config string Configuration file to read from. @@ -163,19 +223,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.3.1/pilosa-v0.3.1-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.4.0/pilosa-v0.4.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.3.1-linux-amd64.tar.gz + tar xfz pilosa-v0.4.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.3.1-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.4.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -190,25 +250,27 @@ There are three ways to install Pilosa on Linux: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. Flags: -c, --config string Configuration file to read from. @@ -249,25 +311,27 @@ There are three ways to install Pilosa on Linux: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. Flags: -c, --config string Configuration file to read from. diff --git a/frame.go b/frame.go index dbc26541e..17abb5f44 100644 --- a/frame.go +++ b/frame.go @@ -34,6 +34,7 @@ const ( DefaultRowLabel = "rowID" DefaultCacheType = CacheTypeRanked DefaultInverseEnabled = false + DefaultRangeEnabled = false // Default ranked frame cache DefaultCacheSize = 50000 @@ -46,6 +47,7 @@ type Frame struct { index string name string timeQuantum TimeQuantum + schema *FrameSchema views map[string]*View @@ -59,6 +61,7 @@ type Frame struct { rowLabel string cacheType string inverseEnabled bool + rangeEnabled bool // Cache size for ranked frames cacheSize uint32 @@ -74,9 +77,10 @@ func NewFrame(path, index, name string) (*Frame, error) { } return &Frame{ - path: path, - index: index, - name: name, + path: path, + index: index, + name: name, + schema: &FrameSchema{}, views: make(map[string]*View), rowAttrStore: NewAttrStore(filepath.Join(path, ".data")), @@ -86,6 +90,7 @@ func NewFrame(path, index, name string) (*Frame, error) { rowLabel: DefaultRowLabel, inverseEnabled: DefaultInverseEnabled, + rangeEnabled: DefaultRangeEnabled, cacheType: DefaultCacheType, cacheSize: DefaultCacheSize, @@ -172,6 +177,11 @@ func (f *Frame) InverseEnabled() bool { return f.inverseEnabled } +// RangeEnabled returns true if range fields can be stored on this frame. +func (f *Frame) RangeEnabled() bool { + return f.rangeEnabled +} + // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 func (f *Frame) SetCacheSize(v uint32) error { @@ -206,6 +216,7 @@ func (f *Frame) Options() FrameOptions { opt := FrameOptions{ RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, + RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: f.timeQuantum, @@ -224,6 +235,8 @@ func (f *Frame) Open() error { if err := f.loadMeta(); err != nil { return err + } else if err := f.loadSchema(); err != nil { + return err } if err := f.openViews(); err != nil { @@ -286,6 +299,7 @@ func (f *Frame) loadMeta() error { f.rowLabel = DefaultRowLabel f.cacheType = DefaultCacheType f.inverseEnabled = DefaultInverseEnabled + f.rangeEnabled = DefaultRangeEnabled f.cacheSize = DefaultCacheSize return nil } else if err != nil { @@ -300,6 +314,7 @@ func (f *Frame) loadMeta() error { f.timeQuantum = TimeQuantum(pb.TimeQuantum) f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled + f.rangeEnabled = pb.RangeEnabled f.cacheSize = pb.CacheSize // Copy cache type. @@ -317,6 +332,7 @@ func (f *Frame) saveMeta() error { buf, err := proto.Marshal(&internal.FrameMeta{ RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, + RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: string(f.timeQuantum), @@ -333,6 +349,35 @@ func (f *Frame) saveMeta() error { return nil } +// loadSchema reads the schema for the frame. +func (f *Frame) loadSchema() error { + buf, err := ioutil.ReadFile(filepath.Join(f.path, ".schema")) + if os.IsNotExist(err) { + f.schema = &FrameSchema{} + return nil + } else if err != nil { + return err + } + + var pb internal.FrameSchema + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + f.schema = decodeFrameSchema(&pb) + + return nil +} + +// saveSchema writes the current schema to disk. +func (f *Frame) saveSchema() error { + if buf, err := proto.Marshal(encodeFrameSchema(f.schema)); err != nil { + return err + } else if err := ioutil.WriteFile(filepath.Join(f.path, ".schema"), buf, 0666); err != nil { + return err + } + return nil +} + // Close closes the frame and its views. func (f *Frame) Close() error { f.mu.Lock() @@ -352,6 +397,13 @@ func (f *Frame) Close() error { return nil } +// Schema returns the frame's current schema. +func (f *Frame) Schema() *FrameSchema { + f.mu.Lock() + defer f.mu.Unlock() + return f.schema +} + // TimeQuantum returns the time quantum for the frame. func (f *Frame) TimeQuantum() TimeQuantum { f.mu.Lock() @@ -619,6 +671,7 @@ func encodeFrame(f *Frame) *internal.Frame { Meta: &internal.FrameMeta{ RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, + RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: string(f.timeQuantum), @@ -648,9 +701,11 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } type FrameOptions struct { RowLabel string `json:"rowLabel,omitempty"` InverseEnabled bool `json:"inverseEnabled,omitempty"` + RangeEnabled bool `json:"rangeEnabled,omitempty"` CacheType string `json:"cacheType,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + Fields []*Field `json:"fields,omitempty"` } // Encode converts o into its internal representation. @@ -658,12 +713,115 @@ func (o *FrameOptions) Encode() *internal.FrameMeta { return &internal.FrameMeta{ RowLabel: o.RowLabel, InverseEnabled: o.InverseEnabled, + RangeEnabled: o.RangeEnabled, CacheType: o.CacheType, CacheSize: o.CacheSize, TimeQuantum: string(o.TimeQuantum), } } +// FrameSchema represents the list of fields on a frame. +type FrameSchema struct { + Fields []*Field +} + +func encodeFrameSchema(schema *FrameSchema) *internal.FrameSchema { + if schema == nil { + return nil + } + return &internal.FrameSchema{ + Fields: encodeFields(schema.Fields), + } +} + +func decodeFrameSchema(schema *internal.FrameSchema) *FrameSchema { + if schema == nil { + return nil + } + return &FrameSchema{ + Fields: decodeFields(schema.Fields), + } +} + +// List of field data types. +const ( + FieldTypeInt = "int" +) + +func IsValidFieldType(v string) bool { + switch v { + case FieldTypeInt: + return true + default: + return false + } +} + +// Field represents a range field on a frame. +type Field struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Min int `json:"min,omitempty"` + Max int `json:"max,omitempty"` +} + +func ValidateField(f *Field) error { + if f.Name == "" { + return ErrFieldNameRequired + } else if !IsValidFieldType(f.Type) { + return ErrInvalidFieldType + } else if f.Min > f.Max { + return ErrInvalidFieldRange + } + return nil +} + +func encodeFields(a []*Field) []*internal.Field { + if len(a) == 0 { + return nil + } + other := make([]*internal.Field, len(a)) + for i := range a { + other[i] = encodeField(a[i]) + } + return other +} + +func decodeFields(a []*internal.Field) []*Field { + if len(a) == 0 { + return nil + } + other := make([]*Field, len(a)) + for i := range a { + other[i] = decodeField(a[i]) + } + return other +} + +func encodeField(f *Field) *internal.Field { + if f == nil { + return nil + } + return &internal.Field{ + Name: f.Name, + Type: f.Type, + Min: int64(f.Min), + Max: int64(f.Max), + } +} + +func decodeField(f *internal.Field) *Field { + if f == nil { + return nil + } + return &Field{ + Name: f.Name, + Type: f.Type, + Min: int(f.Min), + Max: int(f.Max), + } +} + // importBitSet represents slices of row and column ids. // This is used to sort data during import. type importBitSet struct { diff --git a/handler_test.go b/handler_test.go index 1b8c914ad..35d57e912 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1121,8 +1121,8 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) if err != nil { @@ -1150,8 +1150,8 @@ func TestHandler_GetInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { diff --git a/holder_test.go b/holder_test.go index 52288670a..b6864ffc9 100644 --- a/holder_test.go +++ b/holder_test.go @@ -19,13 +19,231 @@ import ( "context" "io/ioutil" "os" + "path/filepath" "reflect" + "strings" "testing" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" ) +func TestHolder_Open(t *testing.T) { + t.Run("ErrIndexName", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err != nil { + t.Fatal(err) + } else if logOutput := h.LogOutput.String(); !strings.Contains(logOutput, `ERROR opening index: !`) { + t.Fatalf("expected log error:\n%s", logOutput) + } + }) + + t.Run("ErrIndexPermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Chmod(h.IndexPath("test"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(h.IndexPath("test"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrIndexMetaCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".meta"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "unexpected EOF") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".data"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=test, err=invalid database") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFramePermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrFrameMetaCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".meta"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=open frame: name=bar, err=unexpected EOF") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrFrameAttrStoreCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".data"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=open frame: name=bar, err=invalid database") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrViewPermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrViewFragmentsMkdir", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFragmentStoragePermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if _, err := view.SetBit(0, 0); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0666) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if _, err := view.SetBit(0, 0); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: slice=0, err=unmarshal storage") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFragmentCachePermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if _, err := view.SetBit(0, 0); err != nil { + t.Fatal(err) + } else if err := view.Fragment(0).FlushCache(); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) +} + // Ensure holder can delete an index and its underlying files. func TestHolder_DeleteIndex(t *testing.T) { hldr := MustOpenHolder() @@ -211,6 +429,23 @@ func (h *Holder) Close() error { return h.Holder.Close() } +// Reopen closes the holder and instantiates and opens a new holder. +func (h *Holder) Reopen() error { + if err := h.Holder.Close(); err != nil { + return err + } + + path, logOutput := h.Path, h.Holder.LogOutput + h.Holder = pilosa.NewHolder() + h.Holder.Path = path + h.Holder.LogOutput = logOutput + if err := h.Holder.Open(); err != nil { + return err + } + + return nil +} + // MustCreateIndexIfNotExists returns a given index. Panic on error. func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOptions) *Index { idx, err := h.Holder.CreateIndexIfNotExists(index, opt) diff --git a/index.go b/index.go index c70c9eec4..69afdbbf9 100644 --- a/index.go +++ b/index.go @@ -413,6 +413,30 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, ErrColumnRowLabelEqual } + // Validate mutually exclusive options if ranges are enabled. + // + // NOTE(https://github.com/pilosa/pilosa/issues/399): + // Cache type should be validated as "none" once it is allowed. + if opt.RangeEnabled { + if opt.InverseEnabled { + return nil, ErrInverseRangeNotAllowed + } else if opt.CacheType != "" && opt.CacheType != CacheTypeLRU { + return nil, ErrRangeCacheNotAllowed + } + opt.CacheSize = 0 + } else { + if len(opt.Fields) > 0 { + return nil, ErrFrameFieldsNotAllowed + } + } + + // Validate fields. + for _, field := range opt.Fields { + if err := ValidateField(field); err != nil { + return nil, err + } + } + // Initialize frame. f, err := i.newFrame(i.FramePath(name), name) if err != nil { @@ -454,6 +478,15 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, err } + // Set schema & save. + f.schema = &FrameSchema{ + Fields: opt.Fields, + } + if err := f.saveSchema(); err != nil { + f.Close() + return nil, err + } + // Add to index's frame lookup. i.frames[name] = f diff --git a/index_test.go b/index_test.go index f4cb534af..49ab014b3 100644 --- a/index_test.go +++ b/index_test.go @@ -17,6 +17,7 @@ package pilosa_test import ( "io/ioutil" "os" + "reflect" "testing" "github.com/pilosa/pilosa" @@ -89,6 +90,123 @@ func TestIndex_CreateFrame(t *testing.T) { }) }) + // Ensure frame can include range columns. + t.Run("RangeEnabled", func(t *testing.T) { + t.Run("OK", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create frame with schema and verify it exists. + if f, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, + }, + }); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, + }, + }) { + t.Fatalf("unexpected schema: %#v", f.Schema()) + } + + // Reopen the index & verify the fields are loaded. + if err := index.Reopen(); err != nil { + t.Fatal(err) + } else if f := index.Frame("f"); !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, + }, + }) { + t.Fatalf("unexpected schema after reopen: %#v", f.Schema()) + } + }) + + t.Run("ErrInverseRangeNotAllowed", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + InverseEnabled: true, + RangeEnabled: true, + }); err != pilosa.ErrInverseRangeNotAllowed { + t.Fatal(err) + } + }) + + t.Run("ErrRangeCacheNotAllowed", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + CacheType: pilosa.CacheTypeRanked, + }); err != pilosa.ErrRangeCacheNotAllowed { + t.Fatal(err) + } + }) + + t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt}, + }, + }); err != pilosa.ErrFrameFieldsNotAllowed { + t.Fatal(err) + } + }) + + t.Run("ErrFieldNameRequired", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "", Type: pilosa.FieldTypeInt}, + }, + }); err != pilosa.ErrFieldNameRequired { + t.Fatal(err) + } + }) + + t.Run("ErrInvalidFieldType", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: "bad_type"}, + }, + }); err != pilosa.ErrInvalidFieldType { + t.Fatal(err) + } + }) + + t.Run("ErrInvalidFieldRange", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, + }, + }); err != pilosa.ErrInvalidFieldRange { + t.Fatal(err) + } + }) + }) + // Ensure frame cannot be created with a matching row label. t.Run("ErrColumnRowLabelEqual", func(t *testing.T) { t.Run("Explicit", func(t *testing.T) { @@ -247,8 +365,8 @@ func TestIndex_CreateInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { @@ -266,8 +384,8 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) if err != nil { @@ -297,8 +415,8 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) if err != nil { @@ -321,8 +439,8 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} input, err := index.CreateInputDefinition(&def) if err != nil { diff --git a/input_definition.go b/input_definition.go index a24163ca6..1e8fd4680 100644 --- a/input_definition.go +++ b/input_definition.go @@ -40,7 +40,7 @@ type InputDefinition struct { index string broadcaster Broadcaster frames []InputFrame - fields []Field + fields []InputDefinitionField } // NewInputDefinition returns a new instance of InputDefinition. @@ -61,7 +61,7 @@ func NewInputDefinition(path, index, name string) (*InputDefinition, error) { func (i *InputDefinition) Frames() []InputFrame { return i.frames } // Fields returns fields of the input definition was initialized with. -func (i *InputDefinition) Fields() []Field { return i.fields } +func (i *InputDefinition) Fields() []InputDefinitionField { return i.fields } // Open opens and initializes the InputDefinition from file. func (i *InputDefinition) Open() error { @@ -103,11 +103,11 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { countRowID := make(map[string]uint64) for _, field := range pb.Fields { var actions []Action - for _, action := range field.Actions { + for _, action := range field.InputDefinitionActions { if err := i.ValidateAction(action); err != nil { return err } - if action.RowID != 0 && action.Frame != "" { + if action.ValueDestination == SingleRowBool && action.Frame != "" { val, ok := countRowID[action.Frame] if ok && val == action.RowID { return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) @@ -130,7 +130,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { return errors.New("duplicate primaryKey with other field") } - inputField := Field{ + inputField := InputDefinitionField{ Name: field.Name, PrimaryKey: field.PrimaryKey, Actions: actions, @@ -175,9 +175,9 @@ func (i *InputDefinition) saveMeta() error { var fields []*internal.InputDefinitionField for _, field := range i.fields { - var actions []*internal.Action + var actions []*internal.InputDefinitionAction for _, action := range field.Actions { - actionMeta := &internal.Action{ + actionMeta := &internal.InputDefinitionAction{ Frame: action.Frame, ValueDestination: action.ValueDestination, ValueMap: action.ValueMap, @@ -187,9 +187,9 @@ func (i *InputDefinition) saveMeta() error { } fieldMeta := &internal.InputDefinitionField{ - Name: field.Name, - PrimaryKey: field.PrimaryKey, - Actions: actions, + Name: field.Name, + PrimaryKey: field.PrimaryKey, + InputDefinitionActions: actions, } fields = append(fields, fieldMeta) } @@ -210,15 +210,15 @@ func (i *InputDefinition) saveMeta() error { return nil } -// Field descripes a single field mapping in the InputDefinition. -type Field struct { +// InputDefinitionField descripes a single field mapping in the InputDefinition. +type InputDefinitionField struct { Name string `json:"name,omitempty"` PrimaryKey bool `json:"primaryKey,omitempty"` Actions []Action `json:"actions,omitempty"` } -// Encode converts Field into its internal representation. -func (o *Field) Encode() (*internal.InputDefinitionField, error) { +// Encode converts InputDefinitionField into its internal representation. +func (o *InputDefinitionField) Encode() (*internal.InputDefinitionField, error) { field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey} for _, action := range o.Actions { @@ -226,7 +226,7 @@ func (o *Field) Encode() (*internal.InputDefinitionField, error) { if err != nil { return nil, err } - field.Actions = append(field.Actions, actionEncode) + field.InputDefinitionActions = append(field.InputDefinitionActions, actionEncode) } return &field, nil } @@ -240,11 +240,11 @@ type Action struct { } // Encode converts Action into its internal representation. -func (o *Action) Encode() (*internal.Action, error) { +func (o *Action) Encode() (*internal.InputDefinitionAction, error) { if o.RowID == nil && o.ValueDestination == "single-row-boolean" { return nil, errors.New("rowID required for single-row-boolean") } - return &internal.Action{ + return &internal.InputDefinitionAction{ Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, @@ -268,7 +268,7 @@ type InputFrame struct { // InputDefinitionInfo the json message format to create an InputDefinition. type InputDefinitionInfo struct { Frames []InputFrame `json:"frames"` - Fields []Field `json:"fields"` + Fields []InputDefinitionField `json:"fields"` } // Encode converts InputDefinitionInfo into its internal representation. @@ -296,7 +296,7 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { return nil } -func (i *InputDefinition) ValidateAction(action *internal.Action) error { +func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) error { if action.Frame == "" { return ErrFrameRequired } diff --git a/input_definition_test.go b/input_definition_test.go index 160064e90..aa6acd5f8 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -29,8 +29,8 @@ func TestInputDefinition_Open(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { @@ -91,10 +91,10 @@ func TestInputDefinition_Encoding(t *testing.T) { t.Fatalf("unexpected frame meta data: %v", internalDef) } else if len(internalDef.Fields) != 2 { t.Fatalf("unexpected number of Fields: %d", len(internalDef.Fields)) - } else if len(internalDef.Fields[1].Actions) != 1 { - t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].Actions) - } else if internalDef.Fields[1].Actions[0].ValueDestination != "mapping" { - t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].Actions[0]) + } else if len(internalDef.Fields[1].InputDefinitionActions) != 1 { + t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].InputDefinitionActions) + } else if internalDef.Fields[1].InputDefinitionActions[0].ValueDestination != "mapping" { + t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].InputDefinitionActions[0]) } } @@ -105,8 +105,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { // Create Input Definition. input := pilosa.InputDefinition{} frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} - field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} + field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err := input.LoadDefinition(def) if !strings.Contains(err.Error(), "invalid ValueDestination") { @@ -119,32 +119,32 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} - field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} + field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "valueMap required for map") { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} - action1 := internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} - field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} + field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "duplicate primaryKey with other field") { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} - field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} + action1 = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "duplicate rowID with other field") { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - action = internal.Action{ValueDestination: pilosa.SingleRowBool, RowID: 100} + action = internal.InputDefinitionAction{ValueDestination: pilosa.SingleRowBool, RowID: 100} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "frame required") { diff --git a/internal/private.pb.go b/internal/private.pb.go index a59d44430..8194a13de 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -25,11 +25,13 @@ Index InputDefinition InputDefinitionField - Action + InputDefinitionAction CreateInputDefinitionMessage DeleteInputDefinitionMessage NodeStatus ClusterStatus + FrameSchema + Field */ package internal @@ -66,6 +68,7 @@ type FrameMeta struct { CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + RangeEnabled bool `protobuf:"varint,6,opt,name=RangeEnabled,proto3" json:"RangeEnabled,omitempty"` } func (m *FrameMeta) Reset() { *m = FrameMeta{} } @@ -273,9 +276,9 @@ func (m *InputDefinition) GetFields() []*InputDefinitionField { } type InputDefinitionField struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - PrimaryKey bool `protobuf:"varint,2,opt,name=PrimaryKey,proto3" json:"PrimaryKey,omitempty"` - Actions []*Action `protobuf:"bytes,3,rep,name=Actions" json:"Actions,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + PrimaryKey bool `protobuf:"varint,2,opt,name=PrimaryKey,proto3" json:"PrimaryKey,omitempty"` + InputDefinitionActions []*InputDefinitionAction `protobuf:"bytes,3,rep,name=InputDefinitionActions" json:"InputDefinitionActions,omitempty"` } func (m *InputDefinitionField) Reset() { *m = InputDefinitionField{} } @@ -283,26 +286,26 @@ func (m *InputDefinitionField) String() string { return proto.Compact func (*InputDefinitionField) ProtoMessage() {} func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } -func (m *InputDefinitionField) GetActions() []*Action { +func (m *InputDefinitionField) GetInputDefinitionActions() []*InputDefinitionAction { if m != nil { - return m.Actions + return m.InputDefinitionActions } return nil } -type Action struct { +type InputDefinitionAction struct { Frame string `protobuf:"bytes,1,opt,name=Frame,proto3" json:"Frame,omitempty"` ValueDestination string `protobuf:"bytes,2,opt,name=ValueDestination,proto3" json:"ValueDestination,omitempty"` ValueMap map[string]uint64 `protobuf:"bytes,3,rep,name=ValueMap" json:"ValueMap,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` RowID uint64 `protobuf:"varint,4,opt,name=RowID,proto3" json:"RowID,omitempty"` } -func (m *Action) Reset() { *m = Action{} } -func (m *Action) String() string { return proto.CompactTextString(m) } -func (*Action) ProtoMessage() {} -func (*Action) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (m *InputDefinitionAction) Reset() { *m = InputDefinitionAction{} } +func (m *InputDefinitionAction) String() string { return proto.CompactTextString(m) } +func (*InputDefinitionAction) ProtoMessage() {} +func (*InputDefinitionAction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } -func (m *Action) GetValueMap() map[string]uint64 { +func (m *InputDefinitionAction) GetValueMap() map[string]uint64 { if m != nil { return m.ValueMap } @@ -375,6 +378,34 @@ func (m *ClusterStatus) GetNodes() []*NodeStatus { return nil } +type FrameSchema struct { + Fields []*Field `protobuf:"bytes,1,rep,name=Fields" json:"Fields,omitempty"` +} + +func (m *FrameSchema) Reset() { *m = FrameSchema{} } +func (m *FrameSchema) String() string { return proto.CompactTextString(m) } +func (*FrameSchema) ProtoMessage() {} +func (*FrameSchema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } + +func (m *FrameSchema) GetFields() []*Field { + if m != nil { + return m.Fields + } + return nil +} + +type Field struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` + Min int64 `protobuf:"varint,3,opt,name=Min,proto3" json:"Min,omitempty"` + Max int64 `protobuf:"varint,4,opt,name=Max,proto3" json:"Max,omitempty"` +} + +func (m *Field) Reset() { *m = Field{} } +func (m *Field) String() string { return proto.CompactTextString(m) } +func (*Field) ProtoMessage() {} +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } + func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") @@ -392,11 +423,13 @@ func init() { proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*InputDefinition)(nil), "internal.InputDefinition") proto.RegisterType((*InputDefinitionField)(nil), "internal.InputDefinitionField") - proto.RegisterType((*Action)(nil), "internal.Action") + proto.RegisterType((*InputDefinitionAction)(nil), "internal.InputDefinitionAction") proto.RegisterType((*CreateInputDefinitionMessage)(nil), "internal.CreateInputDefinitionMessage") proto.RegisterType((*DeleteInputDefinitionMessage)(nil), "internal.DeleteInputDefinitionMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") + proto.RegisterType((*FrameSchema)(nil), "internal.FrameSchema") + proto.RegisterType((*Field)(nil), "internal.Field") } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -476,6 +509,16 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) i += copy(dAtA[i:], m.TimeQuantum) } + if m.RangeEnabled { + dAtA[i] = 0x30 + i++ + if m.RangeEnabled { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -1030,8 +1073,8 @@ func (m *InputDefinitionField) MarshalTo(dAtA []byte) (int, error) { } i++ } - if len(m.Actions) > 0 { - for _, msg := range m.Actions { + if len(m.InputDefinitionActions) > 0 { + for _, msg := range m.InputDefinitionActions { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) @@ -1045,7 +1088,7 @@ func (m *InputDefinitionField) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Action) Marshal() (dAtA []byte, err error) { +func (m *InputDefinitionAction) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1055,7 +1098,7 @@ func (m *Action) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Action) MarshalTo(dAtA []byte) (int, error) { +func (m *InputDefinitionAction) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1238,6 +1281,76 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *FrameSchema) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FrameSchema) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Fields) > 0 { + for _, msg := range m.Fields { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *Field) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Field) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Type) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) + i += copy(dAtA[i:], m.Type) + } + if m.Min != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) + } + if m.Max != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) + } + return i, nil +} + func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) dAtA[offset+1] = uint8(v >> 8) @@ -1300,6 +1413,9 @@ func (m *FrameMeta) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.RangeEnabled { + n += 2 + } return n } @@ -1538,8 +1654,8 @@ func (m *InputDefinitionField) Size() (n int) { if m.PrimaryKey { n += 2 } - if len(m.Actions) > 0 { - for _, e := range m.Actions { + if len(m.InputDefinitionActions) > 0 { + for _, e := range m.InputDefinitionActions { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } @@ -1547,7 +1663,7 @@ func (m *InputDefinitionField) Size() (n int) { return n } -func (m *Action) Size() (n int) { +func (m *InputDefinitionAction) Size() (n int) { var l int _ = l l = len(m.Frame) @@ -1636,6 +1752,38 @@ func (m *ClusterStatus) Size() (n int) { return n } +func (m *FrameSchema) Size() (n int) { + var l int + _ = l + if len(m.Fields) > 0 { + for _, e := range m.Fields { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + +func (m *Field) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Type) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Min != 0 { + n += 1 + sovPrivate(uint64(m.Min)) + } + if m.Max != 0 { + n += 1 + sovPrivate(uint64(m.Max)) + } + return n +} + func sovPrivate(x uint64) (n int) { for { n++ @@ -1912,6 +2060,26 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { } m.TimeQuantum = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RangeEnabled", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.RangeEnabled = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3775,7 +3943,7 @@ func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { m.PrimaryKey = bool(v != 0) case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Actions", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field InputDefinitionActions", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -3799,8 +3967,8 @@ func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Actions = append(m.Actions, &Action{}) - if err := m.Actions[len(m.Actions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.InputDefinitionActions = append(m.InputDefinitionActions, &InputDefinitionAction{}) + if err := m.InputDefinitionActions[len(m.InputDefinitionActions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -3825,7 +3993,7 @@ func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { } return nil } -func (m *Action) Unmarshal(dAtA []byte) error { +func (m *InputDefinitionAction) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3848,10 +4016,10 @@ func (m *Action) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Action: wiretype end group for non-group") + return fmt.Errorf("proto: InputDefinitionAction: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Action: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: InputDefinitionAction: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -4527,6 +4695,233 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } return nil } +func (m *FrameSchema) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FrameSchema: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FrameSchema: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Fields = append(m.Fields, &Field{}) + if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Field) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Field: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Field: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Type = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Min", wireType) + } + m.Min = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Min |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) + } + m.Max = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Max |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -4635,58 +5030,63 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 842 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcd, 0x8e, 0x1b, 0x45, - 0x10, 0x66, 0xec, 0xb1, 0x63, 0xd7, 0x6a, 0x77, 0x9d, 0x66, 0x85, 0x26, 0xab, 0x95, 0xb5, 0xea, - 0x03, 0x59, 0x7c, 0xf0, 0x21, 0x48, 0x08, 0x12, 0x0e, 0x10, 0xdb, 0xd1, 0x5a, 0xe0, 0x00, 0xed, - 0x28, 0x47, 0xa4, 0x5e, 0xbb, 0x80, 0xd1, 0x8e, 0x67, 0xcc, 0x74, 0x8f, 0x37, 0xe6, 0xc0, 0x05, - 0x89, 0x67, 0x40, 0xe2, 0x19, 0x78, 0x0f, 0x8e, 0x5c, 0xb9, 0xa1, 0xe5, 0xc2, 0x63, 0xa0, 0xfe, - 0x9b, 0x19, 0x8f, 0xd7, 0x8e, 0x02, 0xb7, 0xae, 0xaf, 0xbe, 0xae, 0xfa, 0xba, 0xa6, 0xaa, 0x6c, - 0x38, 0x5c, 0xa6, 0xe1, 0x8a, 0x4b, 0xec, 0x2f, 0xd3, 0x44, 0x26, 0xa4, 0x15, 0xc6, 0x12, 0xd3, - 0x98, 0x47, 0xf4, 0x0b, 0x68, 0x8f, 0xe3, 0x39, 0xbe, 0x9a, 0xa0, 0xe4, 0xe4, 0x1c, 0x0e, 0x06, - 0x49, 0x94, 0x2d, 0xe2, 0xcf, 0xf9, 0x15, 0x46, 0x81, 0x77, 0xee, 0x5d, 0xb4, 0x59, 0x19, 0x52, - 0x8c, 0x17, 0xe1, 0x02, 0xbf, 0xca, 0x78, 0x2c, 0xb3, 0x45, 0x50, 0x33, 0x8c, 0x12, 0x44, 0x7f, - 0xf3, 0xa0, 0xfd, 0x2c, 0xe5, 0x0b, 0xd4, 0x11, 0x4f, 0xa1, 0xc5, 0x92, 0x9b, 0x72, 0xb8, 0xdc, - 0x26, 0xef, 0xc2, 0xd1, 0x38, 0x5e, 0x61, 0x2a, 0x70, 0x14, 0xf3, 0xab, 0x08, 0xe7, 0x3a, 0x5c, - 0x8b, 0x55, 0x50, 0x72, 0x06, 0xed, 0x01, 0x9f, 0x7d, 0x87, 0x2f, 0xd6, 0x4b, 0x0c, 0xea, 0x3a, - 0x48, 0x01, 0xe4, 0xde, 0x69, 0xf8, 0x03, 0x06, 0xfe, 0xb9, 0x77, 0x71, 0xc8, 0x0a, 0xa0, 0xaa, - 0xb7, 0xb1, 0xad, 0x97, 0xc2, 0xd1, 0x78, 0xb1, 0x4c, 0x52, 0xc9, 0x50, 0x2c, 0x93, 0x58, 0x20, - 0xe9, 0x40, 0x7d, 0x94, 0xa6, 0x56, 0xae, 0x3a, 0xd2, 0x1f, 0xa1, 0xf3, 0x34, 0x4a, 0x66, 0xd7, - 0x43, 0x2e, 0x39, 0xc3, 0xef, 0x33, 0x14, 0x92, 0x9c, 0x40, 0x43, 0x17, 0xce, 0xf2, 0x8c, 0xa1, - 0x50, 0xfd, 0x78, 0x5b, 0x19, 0x63, 0x28, 0x54, 0xdf, 0xd7, 0xea, 0x7d, 0x66, 0x0c, 0x85, 0x4e, - 0xa3, 0x70, 0x66, 0x54, 0xfb, 0xcc, 0x18, 0x84, 0x80, 0xff, 0x32, 0xc4, 0x1b, 0x2b, 0x55, 0x9f, - 0xe9, 0x18, 0xee, 0x97, 0xf2, 0x5b, 0x99, 0xef, 0x40, 0x93, 0x25, 0x37, 0xe3, 0xa1, 0x08, 0xbc, - 0xf3, 0xfa, 0x85, 0xcf, 0xac, 0xa5, 0x0b, 0xa2, 0xbf, 0x98, 0x72, 0xd5, 0xb4, 0xab, 0x00, 0xe8, - 0x03, 0x68, 0xe8, 0xea, 0xa8, 0x57, 0x16, 0x77, 0xd5, 0x91, 0xfe, 0xea, 0xc1, 0xfd, 0x09, 0x7f, - 0xa5, 0x65, 0x88, 0x3c, 0xcd, 0x25, 0xb4, 0x73, 0x50, 0xb3, 0x0f, 0x1e, 0xf5, 0xfa, 0xae, 0x7d, - 0xfa, 0x5b, 0xfc, 0x02, 0x19, 0xc5, 0x32, 0x5d, 0xb3, 0xe2, 0xf2, 0xe9, 0xc7, 0x70, 0xb4, 0xe9, - 0x54, 0x1a, 0xae, 0x71, 0xed, 0x2a, 0x7d, 0x8d, 0x6b, 0x55, 0x93, 0x15, 0x8f, 0x32, 0x53, 0x3f, - 0x9f, 0x19, 0xe3, 0x71, 0xed, 0x43, 0x8f, 0x7e, 0x0d, 0x64, 0x90, 0x22, 0x97, 0xa8, 0x03, 0x4c, - 0x50, 0x08, 0xfe, 0x2d, 0xee, 0xfe, 0x0a, 0xa6, 0xb2, 0xb5, 0x72, 0x65, 0xcf, 0xa0, 0x3d, 0x16, - 0xb6, 0xb7, 0xf4, 0x97, 0x68, 0xb1, 0x02, 0xa0, 0x3d, 0x20, 0x43, 0x8c, 0x50, 0xa2, 0x1d, 0x87, - 0x3d, 0xf1, 0xe9, 0xd4, 0x69, 0x79, 0x3d, 0x97, 0x3c, 0x04, 0x5f, 0x4d, 0x82, 0x96, 0x72, 0xf0, - 0xe8, 0xed, 0xa2, 0x74, 0xf9, 0xd8, 0x31, 0x4d, 0xa0, 0xa1, 0x0b, 0x6a, 0xa7, 0xe7, 0x35, 0x0f, - 0xbc, 0xa3, 0xcd, 0x5c, 0xaa, 0x7a, 0x35, 0x55, 0x3e, 0x8f, 0x36, 0xd5, 0x27, 0xee, 0xad, 0xff, - 0x35, 0x15, 0x1d, 0x5a, 0x54, 0xb5, 0xeb, 0x73, 0xe5, 0x35, 0x77, 0xf4, 0x79, 0xf7, 0x93, 0xab, - 0x3a, 0xfe, 0xf1, 0x6c, 0xca, 0x37, 0x0b, 0x53, 0xa9, 0x9c, 0x5a, 0x32, 0xae, 0xb1, 0xec, 0x84, - 0xe5, 0x36, 0x79, 0x08, 0x4d, 0x9d, 0x55, 0x04, 0xbe, 0xee, 0xdd, 0xe3, 0x8a, 0x1a, 0x66, 0xdd, - 0x6a, 0x9c, 0x6c, 0x93, 0x37, 0xcc, 0x38, 0x19, 0x8b, 0x8c, 0xa0, 0x33, 0x8e, 0x97, 0x99, 0x1c, - 0xe2, 0x37, 0x61, 0x1c, 0xca, 0x30, 0x89, 0x45, 0xd0, 0xd4, 0xa1, 0x1e, 0x94, 0x15, 0x6d, 0x30, - 0xd8, 0xd6, 0x15, 0xfa, 0xb3, 0x07, 0xc7, 0x15, 0x70, 0xc7, 0xa3, 0x9d, 0xde, 0xda, 0x7e, 0xbd, - 0x1f, 0x40, 0xf3, 0x59, 0x88, 0xd1, 0x5c, 0x04, 0x75, 0x4d, 0xec, 0xee, 0x54, 0xa3, 0x69, 0xcc, - 0xb2, 0xe9, 0x0a, 0x4e, 0xee, 0xf2, 0xdf, 0x29, 0xa6, 0x0b, 0xf0, 0x65, 0x1a, 0x2e, 0x78, 0xba, - 0xfe, 0x0c, 0xd7, 0x76, 0x3b, 0x97, 0x10, 0xd2, 0x83, 0x7b, 0x9f, 0xce, 0x4c, 0x49, 0x8c, 0x88, - 0x4e, 0x21, 0xc2, 0x38, 0x98, 0x23, 0xd0, 0x3f, 0x3d, 0x68, 0x9a, 0x73, 0xd1, 0x52, 0x5e, 0xb9, - 0x7b, 0x7b, 0xd0, 0x79, 0xa9, 0xa6, 0x7d, 0x88, 0x42, 0x86, 0x31, 0x57, 0x4c, 0xdb, 0x73, 0x5b, - 0x38, 0x79, 0x0c, 0x2d, 0x8d, 0x4d, 0xf8, 0x72, 0xfb, 0xf9, 0x26, 0x4b, 0xdf, 0x11, 0xcc, 0x1e, - 0xca, 0xf9, 0x2a, 0xbb, 0xde, 0x94, 0x6e, 0xed, 0x6a, 0xe3, 0xf4, 0x09, 0x1c, 0x6e, 0x5c, 0x78, - 0xa3, 0xdd, 0xf4, 0x93, 0x07, 0x67, 0x6e, 0x21, 0x6c, 0x94, 0x76, 0xff, 0x68, 0xb9, 0x92, 0xd7, - 0x4a, 0x25, 0xff, 0x08, 0xa0, 0xb8, 0x6e, 0x27, 0x79, 0x4f, 0xa3, 0x95, 0xc8, 0xf4, 0x12, 0xce, - 0xdc, 0x06, 0xfb, 0x7f, 0x22, 0x28, 0x07, 0x78, 0x9e, 0xcc, 0x71, 0x2a, 0xb9, 0xcc, 0x84, 0x62, - 0x5c, 0x26, 0x42, 0xba, 0xce, 0x50, 0x67, 0xbd, 0x61, 0x25, 0x97, 0xf9, 0x56, 0xd0, 0x06, 0x79, - 0x0f, 0xee, 0xe9, 0xa0, 0xe8, 0xfa, 0xe1, 0xb8, 0x32, 0xb4, 0xcc, 0xf9, 0xe9, 0x13, 0x38, 0x1c, - 0x44, 0x99, 0x90, 0x98, 0xda, 0x2c, 0x3d, 0x68, 0xa8, 0x9c, 0xee, 0x37, 0xe6, 0xa4, 0xb8, 0x59, - 0x48, 0x61, 0x86, 0xf2, 0xb4, 0xf3, 0xfb, 0x6d, 0xd7, 0xfb, 0xe3, 0xb6, 0xeb, 0xfd, 0x75, 0xdb, - 0xf5, 0x7e, 0xf9, 0xbb, 0xfb, 0xd6, 0x55, 0x53, 0xff, 0xaf, 0x79, 0xff, 0xdf, 0x00, 0x00, 0x00, - 0xff, 0xff, 0x41, 0x4c, 0x6a, 0xbc, 0xe8, 0x08, 0x00, 0x00, + // 915 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcd, 0x6e, 0x23, 0x45, + 0x10, 0xa6, 0xed, 0xb1, 0xb1, 0x2b, 0x24, 0xf1, 0x36, 0x61, 0xe5, 0x8d, 0x22, 0x13, 0xf5, 0x81, + 0x0d, 0x91, 0xc8, 0x61, 0x91, 0x56, 0xc0, 0x72, 0x80, 0x8d, 0xb3, 0x8a, 0x05, 0x5e, 0xa0, 0xbd, + 0x5a, 0x6e, 0x48, 0x1d, 0xa7, 0xd8, 0x1d, 0x65, 0x3c, 0x63, 0x66, 0xda, 0x49, 0xcc, 0x81, 0x0b, + 0x12, 0xcf, 0x80, 0xc4, 0x91, 0x97, 0xe1, 0x08, 0x6f, 0x80, 0xc2, 0x85, 0x37, 0xe0, 0x8a, 0xba, + 0xba, 0x7b, 0x66, 0x3c, 0x8e, 0x13, 0x2d, 0xdc, 0xba, 0xbe, 0xae, 0x9f, 0xaf, 0xaa, 0xab, 0x6a, + 0x06, 0xd6, 0xa7, 0x69, 0x78, 0xae, 0x34, 0x1e, 0x4c, 0xd3, 0x44, 0x27, 0xbc, 0x15, 0xc6, 0x1a, + 0xd3, 0x58, 0x45, 0xe2, 0x0b, 0x68, 0x0f, 0xe2, 0x53, 0xbc, 0x1c, 0xa2, 0x56, 0x7c, 0x17, 0xd6, + 0x0e, 0x93, 0x68, 0x36, 0x89, 0x3f, 0x57, 0x27, 0x18, 0x75, 0xd9, 0x2e, 0xdb, 0x6b, 0xcb, 0x32, + 0x64, 0x34, 0x9e, 0x85, 0x13, 0xfc, 0x6a, 0xa6, 0x62, 0x3d, 0x9b, 0x74, 0x6b, 0x56, 0xa3, 0x04, + 0x89, 0x3f, 0x18, 0xb4, 0x9f, 0xa4, 0x6a, 0x82, 0xe4, 0x71, 0x1b, 0x5a, 0x32, 0xb9, 0x28, 0xbb, + 0xcb, 0x65, 0xfe, 0x0e, 0x6c, 0x0c, 0xe2, 0x73, 0x4c, 0x33, 0x3c, 0x8a, 0xd5, 0x49, 0x84, 0xa7, + 0xe4, 0xae, 0x25, 0x2b, 0x28, 0xdf, 0x81, 0xf6, 0xa1, 0x1a, 0xbf, 0xc4, 0x67, 0xf3, 0x29, 0x76, + 0xeb, 0xe4, 0xa4, 0x00, 0xf2, 0xdb, 0x51, 0xf8, 0x3d, 0x76, 0x83, 0x5d, 0xb6, 0xb7, 0x2e, 0x0b, + 0xa0, 0xca, 0xb7, 0xb1, 0xc4, 0x97, 0x0b, 0x78, 0x43, 0xaa, 0xf8, 0x45, 0xce, 0xa1, 0x49, 0x1c, + 0x16, 0x30, 0x21, 0x60, 0x63, 0x30, 0x99, 0x26, 0xa9, 0x96, 0x98, 0x4d, 0x93, 0x38, 0x43, 0xde, + 0x81, 0xfa, 0x51, 0x9a, 0xba, 0x94, 0xcc, 0x51, 0xfc, 0x00, 0x9d, 0xc7, 0x51, 0x32, 0x3e, 0xeb, + 0x2b, 0xad, 0x24, 0x7e, 0x37, 0xc3, 0x4c, 0xf3, 0x2d, 0x68, 0x50, 0x71, 0x9d, 0x9e, 0x15, 0x0c, + 0x4a, 0x05, 0x72, 0xd5, 0xb3, 0x82, 0x41, 0xc9, 0x9e, 0x32, 0x0c, 0xa4, 0x15, 0x0c, 0x3a, 0x8a, + 0xc2, 0xb1, 0xcd, 0x2c, 0x90, 0x56, 0xe0, 0x1c, 0x82, 0xe7, 0x21, 0x5e, 0xb8, 0x74, 0xe8, 0x2c, + 0x06, 0x70, 0xa7, 0x14, 0xdf, 0xd1, 0xbc, 0x0b, 0x4d, 0x99, 0x5c, 0x0c, 0xfa, 0x59, 0x97, 0xed, + 0xd6, 0xf7, 0x02, 0xe9, 0x24, 0x2a, 0x1a, 0xbd, 0xaa, 0xb9, 0xaa, 0xd1, 0x55, 0x01, 0x88, 0x7b, + 0xd0, 0xa0, 0x0a, 0x9a, 0x2c, 0x0b, 0x5b, 0x73, 0x14, 0xbf, 0x30, 0xb8, 0x33, 0x54, 0x97, 0x44, + 0x23, 0xcb, 0xc3, 0x1c, 0x43, 0x3b, 0x07, 0x49, 0x7b, 0xed, 0xc1, 0xfe, 0x81, 0x6f, 0xb1, 0x83, + 0x25, 0xfd, 0x02, 0x39, 0x8a, 0x75, 0x3a, 0x97, 0x85, 0xf1, 0xf6, 0xc7, 0xb0, 0xb1, 0x78, 0x69, + 0x38, 0x9c, 0xe1, 0xdc, 0x57, 0xfa, 0x0c, 0xe7, 0xa6, 0x26, 0xe7, 0x2a, 0x9a, 0xd9, 0xfa, 0x05, + 0xd2, 0x0a, 0x1f, 0xd5, 0x3e, 0x60, 0xe2, 0x1b, 0xe0, 0x87, 0x29, 0x2a, 0x8d, 0xe4, 0x60, 0x88, + 0x59, 0xa6, 0x5e, 0xe0, 0xea, 0x57, 0xb0, 0x95, 0xad, 0x95, 0x2b, 0xbb, 0x03, 0xed, 0x41, 0xe6, + 0xfa, 0x8f, 0x5e, 0xa2, 0x25, 0x0b, 0x40, 0xec, 0x03, 0xef, 0x63, 0x84, 0x1a, 0xdd, 0xc8, 0xdc, + 0xe0, 0x5f, 0x8c, 0x3c, 0x97, 0xdb, 0x75, 0xf9, 0x7d, 0x08, 0xcc, 0xb4, 0x10, 0x95, 0xb5, 0x07, + 0x6f, 0x16, 0xa5, 0xcb, 0x47, 0x53, 0x92, 0x82, 0x08, 0xbd, 0x53, 0x37, 0x61, 0xb7, 0x24, 0x78, + 0x4d, 0x9b, 0xf9, 0x50, 0xf5, 0x6a, 0xa8, 0x7c, 0x66, 0x5d, 0xa8, 0x4f, 0x7c, 0xae, 0xff, 0x35, + 0x94, 0xe8, 0x3b, 0xd4, 0xb4, 0xeb, 0x53, 0x73, 0x6b, 0x6d, 0xe8, 0xbc, 0x3a, 0xe5, 0x2a, 0x8f, + 0xbf, 0x99, 0x0b, 0xf9, 0x6a, 0x6e, 0x2a, 0x95, 0x33, 0x8b, 0xc8, 0x37, 0x96, 0x9b, 0xb0, 0x5c, + 0xe6, 0xf7, 0xa1, 0x49, 0x51, 0xb3, 0x6e, 0x40, 0xbd, 0xbb, 0x59, 0x61, 0x23, 0xdd, 0xb5, 0x19, + 0x27, 0xd7, 0xe4, 0x0d, 0x3b, 0x4e, 0x56, 0xe2, 0x47, 0xd0, 0x19, 0xc4, 0xd3, 0x99, 0xee, 0xe3, + 0xb7, 0x61, 0x1c, 0xea, 0x30, 0x89, 0xb3, 0x6e, 0x93, 0x5c, 0xdd, 0x2b, 0x33, 0x5a, 0xd0, 0x90, + 0x4b, 0x26, 0xe2, 0x27, 0x06, 0x9b, 0x15, 0x70, 0x45, 0xd2, 0x9e, 0x6f, 0xed, 0x66, 0xbe, 0x0f, + 0xa1, 0xf9, 0x24, 0xc4, 0xe8, 0x34, 0xeb, 0xd6, 0x49, 0xb1, 0xb7, 0x92, 0x0d, 0xa9, 0x49, 0xa7, + 0x2d, 0x7e, 0x65, 0xb0, 0x75, 0x9d, 0xc2, 0xb5, 0x6c, 0x7a, 0x00, 0x5f, 0xa6, 0xe1, 0x44, 0xa5, + 0xf3, 0xcf, 0x70, 0xee, 0x56, 0x78, 0x09, 0xe1, 0x5f, 0xc3, 0xdd, 0x8a, 0xaf, 0x4f, 0xc7, 0xb6, + 0x44, 0x96, 0xd4, 0xdb, 0x2b, 0x49, 0x59, 0x3d, 0xb9, 0xc2, 0x5c, 0xfc, 0xc3, 0xe0, 0xad, 0x6b, + 0xaf, 0x8a, 0x7e, 0x64, 0xe5, 0xd6, 0xdf, 0x87, 0xce, 0x73, 0xb3, 0x2a, 0xfa, 0x98, 0xe9, 0x30, + 0x56, 0x46, 0xd3, 0x35, 0xec, 0x12, 0xce, 0x07, 0xd0, 0x22, 0x6c, 0xa8, 0xa6, 0x8e, 0xe6, 0x7b, + 0xb7, 0xd0, 0x3c, 0xf0, 0xfa, 0x76, 0xa7, 0xe5, 0xe6, 0x86, 0x0c, 0x6d, 0x5d, 0xbf, 0xc2, 0x49, + 0xd8, 0x7e, 0x04, 0xeb, 0x0b, 0x06, 0xaf, 0xb4, 0xe7, 0x7e, 0x64, 0xb0, 0xe3, 0x97, 0xcb, 0x02, + 0x95, 0x9b, 0xc7, 0xd4, 0xbf, 0x5e, 0xad, 0xf4, 0x7a, 0x1f, 0x02, 0x14, 0xe6, 0x6e, 0x2b, 0xdc, + 0xd0, 0xb4, 0x25, 0x65, 0x71, 0x0c, 0x3b, 0x7e, 0x1b, 0xfe, 0x3f, 0x12, 0x42, 0x01, 0x3c, 0x4d, + 0x4e, 0x71, 0xa4, 0x95, 0x9e, 0x65, 0x46, 0xe3, 0x38, 0xc9, 0xb4, 0x6f, 0x32, 0x73, 0xa6, 0x6d, + 0xad, 0x95, 0xce, 0x37, 0x0c, 0x09, 0xfc, 0x5d, 0x78, 0x9d, 0x9c, 0xa2, 0xef, 0xa5, 0xcd, 0xca, + 0x02, 0x90, 0xfe, 0x5e, 0x3c, 0x82, 0xf5, 0xc3, 0x68, 0x96, 0x69, 0x4c, 0x5d, 0x94, 0x7d, 0x68, + 0x98, 0x98, 0xfe, 0x7b, 0xb5, 0x55, 0x58, 0x16, 0x54, 0xa4, 0x55, 0x11, 0x0f, 0x61, 0x8d, 0x5a, + 0x68, 0x34, 0x7e, 0x89, 0x13, 0x45, 0xf3, 0x67, 0xc7, 0x8a, 0x2d, 0xcd, 0xdf, 0xc2, 0x1c, 0x8d, + 0xa0, 0xb1, 0x7a, 0x6e, 0x38, 0x04, 0xf4, 0x47, 0xe3, 0x0a, 0x41, 0x3f, 0x33, 0x1d, 0xa8, 0x0f, + 0x43, 0xfb, 0x0c, 0x75, 0x69, 0x8e, 0x84, 0xa8, 0x4b, 0xea, 0x1d, 0x83, 0xa8, 0xcb, 0xc7, 0x9d, + 0xdf, 0xae, 0x7a, 0xec, 0xf7, 0xab, 0x1e, 0xfb, 0xf3, 0xaa, 0xc7, 0x7e, 0xfe, 0xab, 0xf7, 0xda, + 0x49, 0x93, 0x7e, 0xea, 0xde, 0xff, 0x37, 0x00, 0x00, 0xff, 0xff, 0xb5, 0x12, 0x7e, 0xdc, 0xe5, + 0x09, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 3ad12df7c..f69b0de7b 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -13,6 +13,7 @@ message FrameMeta { string CacheType = 3; uint32 CacheSize = 4; string TimeQuantum = 5; + bool RangeEnabled = 6; } message ImportResponse { @@ -89,10 +90,10 @@ message InputDefinition { message InputDefinitionField { string Name = 1; bool PrimaryKey = 2; - repeated Action Actions = 3; + repeated InputDefinitionAction InputDefinitionActions = 3; } -message Action { +message InputDefinitionAction { string Frame = 1; string ValueDestination = 2; map ValueMap = 3; @@ -120,3 +121,14 @@ message NodeStatus { message ClusterStatus { repeated NodeStatus Nodes = 1; } + +message FrameSchema { + repeated Field Fields = 1; +} + +message Field { + string Name = 1; + string Type = 2; + int64 Min = 3; + int64 Max = 4; +} diff --git a/pilosa.go b/pilosa.go index ff1ef148b..3edcbfad9 100644 --- a/pilosa.go +++ b/pilosa.go @@ -39,6 +39,13 @@ var ( ErrInputDefinitionExists = errors.New("input-definition already exists") ErrInputDefinitionNotFound = errors.New("input-definition not found") + ErrFieldNameRequired = errors.New("field name required") + ErrInvalidFieldType = errors.New("invalid field type") + ErrInvalidFieldRange = errors.New("invalid field range") + ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") + ErrRangeCacheNotAllowed = errors.New("range cache not allowed") + ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") + ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") diff --git a/server.go b/server.go index 9a4c5d4f1..025933cd4 100644 --- a/server.go +++ b/server.go @@ -24,8 +24,10 @@ import ( "net/http" "net/url" "os" + "os/exec" "runtime" "strconv" + "strings" "sync" "time" @@ -475,6 +477,7 @@ func (s *Server) monitorRuntime() { return } + var m runtime.MemStats ticker := time.NewTicker(s.MetricInterval) defer ticker.Stop() @@ -496,9 +499,43 @@ func (s *Server) monitorRuntime() { // Record the number of go routines s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) + + // Open File handles + s.Holder.Stats.Gauge("OpenFiles", float64(CountOpenFiles()), 1.0) + + // Runtime memory metrics + runtime.ReadMemStats(&m) + s.Holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0) + s.Holder.Stats.Gauge("HeapInuse", float64(m.HeapInuse), 1.0) + s.Holder.Stats.Gauge("StackInuse", float64(m.StackInuse), 1.0) + s.Holder.Stats.Gauge("Mallocs", float64(m.Mallocs), 1.0) + s.Holder.Stats.Gauge("Frees", float64(m.Frees), 1.0) } } +// CountOpenFiles on opperating systems that support lsof +func CountOpenFiles() int { + count := 0 + + switch runtime.GOOS { + case "darwin", "linux", "unix", "freebsd": + // -b option avoid kernel blocks + pid := os.Getpid() + out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -b -p %v", pid)).Output() + if err != nil { + log.Fatal(err) + } + // only count lines with our pid, avoiding warning messages from -b + lines := strings.Split(string(out), strconv.Itoa(pid)) + count = len(lines) + case "windows": + // TODO: count open file handles on windows + default: + + } + return count +} + // StatusHandler specifies two methods which an object must implement to share // state in the cluster. These are used by the GossipNodeSet to implement the // LocalState and MergeRemoteState methods of memberlist.Delegate diff --git a/server/server_test.go b/server/server_test.go index a92da70a8..6f4570709 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -25,7 +25,9 @@ import ( "net" "net/http" "os" + "path/filepath" "reflect" + "runtime" "sort" "strconv" "strings" @@ -372,6 +374,42 @@ path = "/path/to/plugins" } } +// tempMkdir makes a temporary directory +func tempMkdir(t *testing.T) string { + dir, err := ioutil.TempDir("", "pilosatemp") + if err != nil { + t.Fatalf("failed to create test directory: %s", err) + } + return dir +} + +// Ensure the file handle count is working +func TestCountOpenFiles(t *testing.T) { + // Windows is not supported yet + supported := []string{"darwin", "linux", "unix", "freebsd"} + sort.Strings(supported) + i := sort.Search(len(supported), + func(i int) bool { return supported[i] >= runtime.GOOS }) + if i == len(supported) { + return + } + + // Create directory store temp file + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + count := pilosa.CountOpenFiles() + testFile := filepath.Join(testDir, "test.txt") + _, err := os.Create(testFile) + if err != nil { + t.Fatalf("create test file failed: %s", err) + } + + if pilosa.CountOpenFiles() < count+1 { + t.Error("Invalid open file handle count") + } +} + // Ensure program can send/receive broadcast messages. func TestMain_SendReceiveMessage(t *testing.T) { m0 := MustRunMain() diff --git a/view.go b/view.go index 5c6198a5e..367a69b5a 100644 --- a/view.go +++ b/view.go @@ -144,7 +144,7 @@ func (v *View) openFragments() error { frag := v.newFragment(v.FragmentPath(slice), slice) if err := frag.Open(); err != nil { - return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err) + return fmt.Errorf("open fragment: slice=%d, err=%s", frag.Slice(), err) } frag.RowAttrStore = v.RowAttrStore v.fragments[frag.Slice()] = frag