From 47e83c31c6cc2b4190c68ffc7253a013906c78f5 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 1 Jun 2017 15:50:30 -0500 Subject: [PATCH 01/12] capture runtime memory metrics in StatsD --- server.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server.go b/server.go index 66034e5d6..ef2c09f76 100644 --- a/server.go +++ b/server.go @@ -460,6 +460,7 @@ func (s *Server) monitorRuntime() { return } + var m runtime.MemStats ticker := time.NewTicker(s.MetricInterval) defer ticker.Stop() @@ -481,6 +482,14 @@ func (s *Server) monitorRuntime() { // Record the number of go routines s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) + + // Runtime memory metrics + runtime.ReadMemStats(&m) + s.Holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0) + s.Holder.Stats.Gauge("HeapInuse", float64(m.HeapInuse), 1.0) + s.Holder.Stats.Gauge("StackInuse", float64(m.StackInuse), 1.0) + s.Holder.Stats.Gauge("Mallocs", float64(m.Mallocs), 1.0) + s.Holder.Stats.Gauge("Frees", float64(m.Frees), 1.0) } } From 6b307f32e0b933b93fa3da650a71a6b36785f766 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 7 Jun 2017 15:03:05 -0500 Subject: [PATCH 02/12] make sure glide is installed before `glide install` If glide is not installed, then $(GLIDE) is empty, so I guess the $(GLIDE) target gets ignored? The impetus for this is that `make install` was failing on a fresh ubuntu instance saying glide wasn't found. --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 563059137..ca672df1c 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,9 @@ $(GLIDE): make glide vendor: $(GLIDE) glide.yaml +ifndef GLIDE + curl https://glide.sh/get | sh +endif glide install glide.lock: glide glide.yaml From f8e0f123038d634995c8385b357bb6539a79d9df Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 8 Jun 2017 15:53:54 -0500 Subject: [PATCH 03/12] Pilosa 0.4.0 docs updates and Homebrew installation guide --- docs/api-reference.md | 2 +- docs/installation.md | 184 +++++++++++++++++++++++++++--------------- 2 files changed, 122 insertions(+), 64 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 55bbde24e..39f66683f 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -240,6 +240,6 @@ curl -XGET localhost:10101/version Response: ``` -{"version":"v0.3.0-353-ge633247"} +{"version":"v0.4.0"} ``` diff --git a/docs/installation.md b/docs/installation.md index c97ed4303..8ee8010d7 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -9,25 +9,79 @@ Pilosa is currently available for [MacOS](#installing-on-macos) and [Linux](#ins ### Installing on MacOS -There are three ways to install Pilosa on MacOS: download the binary (recommended), build from source, or use Docker. +There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) (recommended), download the binary, build from source, or use Docker. + +#### Use Homebrew + +1. Update your Homebrew formulas: + ``` + brew update + ``` + +2. Install Pilosa + ``` + brew install pilosa + ``` + +3. Make sure Pilosa is installed successfully: + ``` + pilosa + ``` + + If you see something like: + ``` + Pilosa is a fast index to turbocharge your database. + + This binary contains Pilosa itself, as well as common + tools for administering pilosa, importing/exporting data, + backing up, and more. Complete documentation is available + at https://www.pilosa.com/docs/ + + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 + + Usage: + pilosa [command] + + Available Commands: + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. + + Flags: + -c, --config string Configuration file to read from. + + Use "pilosa [command] --help" for more information about a command. + ``` + + You're good to go! #### Download the Binary 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.3.1/pilosa-v0.3.1-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.4.0/pilosa-v0.4.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.3.1-darwin-amd64.tar.gz + tar xfz pilosa-v0.4.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.3.1-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.4.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -42,26 +96,27 @@ There are three ways to install Pilosa on MacOS: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. - sort Sort import data for optimal import performance. + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. Flags: -c, --config string Configuration file to read from. @@ -102,26 +157,27 @@ There are three ways to install Pilosa on MacOS: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. - sort Sort import data for optimal import performance. + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. Flags: -c, --config string Configuration file to read from. @@ -165,19 +221,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.3.1/pilosa-v0.3.1-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.4.0/pilosa-v0.4.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.3.1-linux-amd64.tar.gz + tar xfz pilosa-v0.4.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.3.1-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.4.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -192,26 +248,27 @@ There are three ways to install Pilosa on Linux: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. - sort Sort import data for optimal import performance. + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. Flags: -c, --config string Configuration file to read from. @@ -252,26 +309,27 @@ There are three ways to install Pilosa on Linux: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. - sort Sort import data for optimal import performance. + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. Flags: -c, --config string Configuration file to read from. From 472e9bd2e924ee4831b026db1b9e9b7021da9ffb Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 9 Jun 2017 12:01:10 -0500 Subject: [PATCH 04/12] Add PULL_REQUEST_TEMPLATE file. --- .github/PULL_REQUEST_TEMPLATE.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..d96b88aeb --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,22 @@ +## Overview + +[Describe what this pull request addresses.] + +Fixes # + +## Pull request checklist + +- [ ] I have updated the [documentation](https://github.com/pilosa/pilosa/tree/master/docs). +- [ ] I have resolved any merge conflicts. +- [ ] I have included tests that cover my changes. +- [ ] All new and existing tests pass. + +## Code review checklist +This is the checklist that the reviewer will follow while reviewing your pull request. You do not need to do anything with this checklist, but be aware of what the reviewer will be looking for. + +- [ ] Ensure that any changes to external docs have been included in this pull request. +- [ ] If the changes require that minor/major versions need to be updated, tag the PR appropriately. +- [ ] Ensure the new code is [properly commented](https://github.com/golang/go/wiki/CodeReviewComments#doc-comments) and follows [Idiomatic Go](https://dmitri.shuralyov.com/idiomatic-go). +- [ ] Check that tests have been written and that they cover the new functionality. +- [ ] Run tests and ensure they pass. +- [ ] Build and run the code, performing any applicable integration testing. From 571d249ea12d86b9e9f666e4f5f589cab377ff3e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 9 Jun 2017 14:35:57 -0500 Subject: [PATCH 05/12] Add contributing guide and CLA info to pull request template --- .github/PULL_REQUEST_TEMPLATE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d96b88aeb..081a20073 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,6 +6,8 @@ Fixes # ## Pull request checklist +- [ ] I have read the [contributing guide](https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md). +- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa). - [ ] I have updated the [documentation](https://github.com/pilosa/pilosa/tree/master/docs). - [ ] I have resolved any merge conflicts. - [ ] I have included tests that cover my changes. From 1e793a1c524179e1d4503ccda1a29dc95360e10e Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 12 Jun 2017 13:16:19 -0500 Subject: [PATCH 06/12] Count open file handles as a StatsD metric. --- server.go | 31 +++++++++++++++++++++++++++++++ server/server_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/server.go b/server.go index f9719c563..7f2027f10 100644 --- a/server.go +++ b/server.go @@ -24,8 +24,10 @@ import ( "net/http" "net/url" "os" + "os/exec" "runtime" "strconv" + "strings" "sync" "time" @@ -486,6 +488,9 @@ func (s *Server) monitorRuntime() { // Record the number of go routines s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) + // Open File handles + s.Holder.Stats.Gauge("OpenFiles", float64(CountOpenFiles()), 1.0) + // Runtime memory metrics runtime.ReadMemStats(&m) s.Holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0) @@ -496,6 +501,32 @@ func (s *Server) monitorRuntime() { } } +// CountOpenFiles on opperating systems that support lsof +func CountOpenFiles() int { + count := 0 + + switch runtime.GOOS { + case "darwin": + fallthrough + case "linux": + fallthrough + case "unix": + fallthrough + case "freebsd": + out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -p %v", os.Getpid())).Output() + if err != nil { + log.Fatal(err) + } + lines := strings.Split(string(out), "\n") + count = len(lines) - 1 + case "windows": + // TODO: count open file handles on windows + default: + + } + return count +} + // StatusHandler specifies two methods which an object must implement to share // state in the cluster. These are used by the GossipNodeSet to implement the // LocalState and MergeRemoteState methods of memberlist.Delegate diff --git a/server/server_test.go b/server/server_test.go index cd82ba007..f1ee11b23 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -25,7 +25,9 @@ import ( "net" "net/http" "os" + "path/filepath" "reflect" + "runtime" "sort" "strconv" "strings" @@ -372,6 +374,42 @@ path = "/path/to/plugins" } } +// tempMkdir makes a temporary directory +func tempMkdir(t *testing.T) string { + dir, err := ioutil.TempDir("", "pilosatemp") + if err != nil { + t.Fatalf("failed to create test directory: %s", err) + } + return dir +} + +// Ensure the file handle count is working +func TestCountOpenFiles(t *testing.T) { + // Windows is not supported yet + supported := []string{"darwin", "linux", "unix", "freebsd"} + sort.Strings(supported) + i := sort.Search(len(supported), + func(i int) bool { return supported[i] >= runtime.GOOS }) + if i == len(supported) { + return + } + + // Create directory store temp file + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + count := pilosa.CountOpenFiles() + testFile := filepath.Join(testDir, "test.txt") + _, err := os.Create(testFile) + if err != nil { + t.Fatalf("create test file failed: %s", err) + } + + if pilosa.CountOpenFiles() < count+1 { + t.Error("Invalid open file handle count") + } +} + // Ensure program can send/receive broadcast messages. func TestMain_SendReceiveMessage(t *testing.T) { m0 := MustRunMain() From 3a43a862ec19e0e75d533d4c9b8110679d28fa78 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sat, 10 Jun 2017 10:31:36 -0600 Subject: [PATCH 07/12] Add Holder test coverage. --- holder_test.go | 235 +++++++++++++++++++++++++++++++++++++++++++++++++ view.go | 2 +- 2 files changed, 236 insertions(+), 1 deletion(-) diff --git a/holder_test.go b/holder_test.go index 52288670a..b6864ffc9 100644 --- a/holder_test.go +++ b/holder_test.go @@ -19,13 +19,231 @@ import ( "context" "io/ioutil" "os" + "path/filepath" "reflect" + "strings" "testing" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" ) +func TestHolder_Open(t *testing.T) { + t.Run("ErrIndexName", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err != nil { + t.Fatal(err) + } else if logOutput := h.LogOutput.String(); !strings.Contains(logOutput, `ERROR opening index: !`) { + t.Fatalf("expected log error:\n%s", logOutput) + } + }) + + t.Run("ErrIndexPermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Chmod(h.IndexPath("test"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(h.IndexPath("test"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrIndexMetaCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".meta"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "unexpected EOF") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".data"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=test, err=invalid database") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFramePermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrFrameMetaCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".meta"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=open frame: name=bar, err=unexpected EOF") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrFrameAttrStoreCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".data"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=open frame: name=bar, err=invalid database") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrViewPermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrViewFragmentsMkdir", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFragmentStoragePermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if _, err := view.SetBit(0, 0); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0666) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if _, err := view.SetBit(0, 0); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: slice=0, err=unmarshal storage") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFragmentCachePermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if _, err := view.SetBit(0, 0); err != nil { + t.Fatal(err) + } else if err := view.Fragment(0).FlushCache(); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) +} + // Ensure holder can delete an index and its underlying files. func TestHolder_DeleteIndex(t *testing.T) { hldr := MustOpenHolder() @@ -211,6 +429,23 @@ func (h *Holder) Close() error { return h.Holder.Close() } +// Reopen closes the holder and instantiates and opens a new holder. +func (h *Holder) Reopen() error { + if err := h.Holder.Close(); err != nil { + return err + } + + path, logOutput := h.Path, h.Holder.LogOutput + h.Holder = pilosa.NewHolder() + h.Holder.Path = path + h.Holder.LogOutput = logOutput + if err := h.Holder.Open(); err != nil { + return err + } + + return nil +} + // MustCreateIndexIfNotExists returns a given index. Panic on error. func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOptions) *Index { idx, err := h.Holder.CreateIndexIfNotExists(index, opt) diff --git a/view.go b/view.go index 5c6198a5e..367a69b5a 100644 --- a/view.go +++ b/view.go @@ -144,7 +144,7 @@ func (v *View) openFragments() error { frag := v.newFragment(v.FragmentPath(slice), slice) if err := frag.Open(); err != nil { - return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err) + return fmt.Errorf("open fragment: slice=%d, err=%s", frag.Slice(), err) } frag.RowAttrStore = v.RowAttrStore v.fragments[frag.Slice()] = frag From 188eaefb575ae8601ea5999a54c7cd93d63bfbf0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 12 Jun 2017 14:21:32 -0500 Subject: [PATCH 08/12] Remove unintentional dash from LDFLAGS --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ca672df1c..a5113c2ab 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa BUILD_TIME=`date -u +%FT%T%z` -LDFLAGS=-"-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" +LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" default: test pilosa From 76357374879c3e4080a9c1fd53c98ab0c689c970 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 12 Jun 2017 15:20:24 -0500 Subject: [PATCH 09/12] Add coverage tools to Makefile `make cover-viz` will now generate an HTML coverage report for the entire codebase. --- Makefile | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a5113c2ab..6ef21fda0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release test +.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release test cover cover-pkg cover-viz GLIDE := $(shell command -v glide 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) @@ -6,6 +6,7 @@ PROTOC := $(shell command -v protoc 2>/dev/null) VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa +PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) BUILD_TIME=`date -u +%FT%T%z` LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" @@ -34,7 +35,23 @@ glide.lock: glide glide.yaml vendor-update: glide.lock test: vendor - go test $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) $(TESTFLAGS) + go test $(PKGS) $(TESTFLAGS) + +cover: vendor + mkdir -p build/coverage + echo "mode: set" > build/coverage/all.out + for pkg in $(PKGS) ; do \ + make cover-pkg PKG=$$pkg ; \ + done + +cover-pkg: + mkdir -p build/coverage + touch build/coverage/$(subst /,-,$(PKG)).out + go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG) + tail +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out + +cover-viz: cover + go tool cover -html=build/coverage/all.out pilosa: vendor go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa From cd532e08cbc188fc089a9e722b25c2a187fe9b12 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 13 Jun 2017 16:14:45 -0500 Subject: [PATCH 10/12] Add -b option to lsof to avoid kernel blocks. --- docs/administration.md | 2 ++ server.go | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 322b68e12..e562f94ce 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -157,3 +157,5 @@ We currently track the following events Garbage Collection: Event count when Garbage Collection occurs. Goroutines: Number of running Goroutines. + +OpenFiles: Number of open file handles associated with running Pilosa process ID. \ No newline at end of file diff --git a/server.go b/server.go index 7f2027f10..57f3291eb 100644 --- a/server.go +++ b/server.go @@ -513,12 +513,15 @@ func CountOpenFiles() int { case "unix": fallthrough case "freebsd": - out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -p %v", os.Getpid())).Output() + // -b option avoid kernel blocks + pid := os.Getpid() + out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -b -p %v", pid)).Output() if err != nil { log.Fatal(err) } - lines := strings.Split(string(out), "\n") - count = len(lines) - 1 + // only count lines with our pid, avoiding warning messages from -b + lines := strings.Split(string(out), strconv.Itoa(pid)) + count = len(lines) case "windows": // TODO: count open file handles on windows default: From 87f294023319fddebcc7ccdba6525dcc33c762e0 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 14 Jun 2017 09:26:13 -0500 Subject: [PATCH 11/12] declare multiple values in the same switch case --- server.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/server.go b/server.go index 57f3291eb..37fadedd5 100644 --- a/server.go +++ b/server.go @@ -506,13 +506,7 @@ func CountOpenFiles() int { count := 0 switch runtime.GOOS { - case "darwin": - fallthrough - case "linux": - fallthrough - case "unix": - fallthrough - case "freebsd": + case "darwin", "linux", "unix", "freebsd": // -b option avoid kernel blocks pid := os.Getpid() out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -b -p %v", pid)).Output() From 0107ddffea943180847d8ff7bbfdbdc97b9059dd Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 20 Jun 2017 21:52:29 -0600 Subject: [PATCH 12/12] BSI range-encoding schema support. --- frame.go | 164 +++++++++++++- index.go | 33 +++ index_test.go | 118 ++++++++++ internal/private.pb.go | 482 +++++++++++++++++++++++++++++++++++++---- internal/private.proto | 12 + internal/public.pb.go | 2 +- pilosa.go | 7 + 7 files changed, 773 insertions(+), 45 deletions(-) diff --git a/frame.go b/frame.go index dbc26541e..17abb5f44 100644 --- a/frame.go +++ b/frame.go @@ -34,6 +34,7 @@ const ( DefaultRowLabel = "rowID" DefaultCacheType = CacheTypeRanked DefaultInverseEnabled = false + DefaultRangeEnabled = false // Default ranked frame cache DefaultCacheSize = 50000 @@ -46,6 +47,7 @@ type Frame struct { index string name string timeQuantum TimeQuantum + schema *FrameSchema views map[string]*View @@ -59,6 +61,7 @@ type Frame struct { rowLabel string cacheType string inverseEnabled bool + rangeEnabled bool // Cache size for ranked frames cacheSize uint32 @@ -74,9 +77,10 @@ func NewFrame(path, index, name string) (*Frame, error) { } return &Frame{ - path: path, - index: index, - name: name, + path: path, + index: index, + name: name, + schema: &FrameSchema{}, views: make(map[string]*View), rowAttrStore: NewAttrStore(filepath.Join(path, ".data")), @@ -86,6 +90,7 @@ func NewFrame(path, index, name string) (*Frame, error) { rowLabel: DefaultRowLabel, inverseEnabled: DefaultInverseEnabled, + rangeEnabled: DefaultRangeEnabled, cacheType: DefaultCacheType, cacheSize: DefaultCacheSize, @@ -172,6 +177,11 @@ func (f *Frame) InverseEnabled() bool { return f.inverseEnabled } +// RangeEnabled returns true if range fields can be stored on this frame. +func (f *Frame) RangeEnabled() bool { + return f.rangeEnabled +} + // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 func (f *Frame) SetCacheSize(v uint32) error { @@ -206,6 +216,7 @@ func (f *Frame) Options() FrameOptions { opt := FrameOptions{ RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, + RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: f.timeQuantum, @@ -224,6 +235,8 @@ func (f *Frame) Open() error { if err := f.loadMeta(); err != nil { return err + } else if err := f.loadSchema(); err != nil { + return err } if err := f.openViews(); err != nil { @@ -286,6 +299,7 @@ func (f *Frame) loadMeta() error { f.rowLabel = DefaultRowLabel f.cacheType = DefaultCacheType f.inverseEnabled = DefaultInverseEnabled + f.rangeEnabled = DefaultRangeEnabled f.cacheSize = DefaultCacheSize return nil } else if err != nil { @@ -300,6 +314,7 @@ func (f *Frame) loadMeta() error { f.timeQuantum = TimeQuantum(pb.TimeQuantum) f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled + f.rangeEnabled = pb.RangeEnabled f.cacheSize = pb.CacheSize // Copy cache type. @@ -317,6 +332,7 @@ func (f *Frame) saveMeta() error { buf, err := proto.Marshal(&internal.FrameMeta{ RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, + RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: string(f.timeQuantum), @@ -333,6 +349,35 @@ func (f *Frame) saveMeta() error { return nil } +// loadSchema reads the schema for the frame. +func (f *Frame) loadSchema() error { + buf, err := ioutil.ReadFile(filepath.Join(f.path, ".schema")) + if os.IsNotExist(err) { + f.schema = &FrameSchema{} + return nil + } else if err != nil { + return err + } + + var pb internal.FrameSchema + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + f.schema = decodeFrameSchema(&pb) + + return nil +} + +// saveSchema writes the current schema to disk. +func (f *Frame) saveSchema() error { + if buf, err := proto.Marshal(encodeFrameSchema(f.schema)); err != nil { + return err + } else if err := ioutil.WriteFile(filepath.Join(f.path, ".schema"), buf, 0666); err != nil { + return err + } + return nil +} + // Close closes the frame and its views. func (f *Frame) Close() error { f.mu.Lock() @@ -352,6 +397,13 @@ func (f *Frame) Close() error { return nil } +// Schema returns the frame's current schema. +func (f *Frame) Schema() *FrameSchema { + f.mu.Lock() + defer f.mu.Unlock() + return f.schema +} + // TimeQuantum returns the time quantum for the frame. func (f *Frame) TimeQuantum() TimeQuantum { f.mu.Lock() @@ -619,6 +671,7 @@ func encodeFrame(f *Frame) *internal.Frame { Meta: &internal.FrameMeta{ RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, + RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: string(f.timeQuantum), @@ -648,9 +701,11 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } type FrameOptions struct { RowLabel string `json:"rowLabel,omitempty"` InverseEnabled bool `json:"inverseEnabled,omitempty"` + RangeEnabled bool `json:"rangeEnabled,omitempty"` CacheType string `json:"cacheType,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + Fields []*Field `json:"fields,omitempty"` } // Encode converts o into its internal representation. @@ -658,12 +713,115 @@ func (o *FrameOptions) Encode() *internal.FrameMeta { return &internal.FrameMeta{ RowLabel: o.RowLabel, InverseEnabled: o.InverseEnabled, + RangeEnabled: o.RangeEnabled, CacheType: o.CacheType, CacheSize: o.CacheSize, TimeQuantum: string(o.TimeQuantum), } } +// FrameSchema represents the list of fields on a frame. +type FrameSchema struct { + Fields []*Field +} + +func encodeFrameSchema(schema *FrameSchema) *internal.FrameSchema { + if schema == nil { + return nil + } + return &internal.FrameSchema{ + Fields: encodeFields(schema.Fields), + } +} + +func decodeFrameSchema(schema *internal.FrameSchema) *FrameSchema { + if schema == nil { + return nil + } + return &FrameSchema{ + Fields: decodeFields(schema.Fields), + } +} + +// List of field data types. +const ( + FieldTypeInt = "int" +) + +func IsValidFieldType(v string) bool { + switch v { + case FieldTypeInt: + return true + default: + return false + } +} + +// Field represents a range field on a frame. +type Field struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Min int `json:"min,omitempty"` + Max int `json:"max,omitempty"` +} + +func ValidateField(f *Field) error { + if f.Name == "" { + return ErrFieldNameRequired + } else if !IsValidFieldType(f.Type) { + return ErrInvalidFieldType + } else if f.Min > f.Max { + return ErrInvalidFieldRange + } + return nil +} + +func encodeFields(a []*Field) []*internal.Field { + if len(a) == 0 { + return nil + } + other := make([]*internal.Field, len(a)) + for i := range a { + other[i] = encodeField(a[i]) + } + return other +} + +func decodeFields(a []*internal.Field) []*Field { + if len(a) == 0 { + return nil + } + other := make([]*Field, len(a)) + for i := range a { + other[i] = decodeField(a[i]) + } + return other +} + +func encodeField(f *Field) *internal.Field { + if f == nil { + return nil + } + return &internal.Field{ + Name: f.Name, + Type: f.Type, + Min: int64(f.Min), + Max: int64(f.Max), + } +} + +func decodeField(f *internal.Field) *Field { + if f == nil { + return nil + } + return &Field{ + Name: f.Name, + Type: f.Type, + Min: int(f.Min), + Max: int(f.Max), + } +} + // importBitSet represents slices of row and column ids. // This is used to sort data during import. type importBitSet struct { diff --git a/index.go b/index.go index 8a1b34aed..a5cdd1911 100644 --- a/index.go +++ b/index.go @@ -387,6 +387,30 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, ErrColumnRowLabelEqual } + // Validate mutually exclusive options if ranges are enabled. + // + // NOTE(https://github.com/pilosa/pilosa/issues/399): + // Cache type should be validated as "none" once it is allowed. + if opt.RangeEnabled { + if opt.InverseEnabled { + return nil, ErrInverseRangeNotAllowed + } else if opt.CacheType != "" && opt.CacheType != CacheTypeLRU { + return nil, ErrRangeCacheNotAllowed + } + opt.CacheSize = 0 + } else { + if len(opt.Fields) > 0 { + return nil, ErrFrameFieldsNotAllowed + } + } + + // Validate fields. + for _, field := range opt.Fields { + if err := ValidateField(field); err != nil { + return nil, err + } + } + // Initialize frame. f, err := i.newFrame(i.FramePath(name), name) if err != nil { @@ -428,6 +452,15 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, err } + // Set schema & save. + f.schema = &FrameSchema{ + Fields: opt.Fields, + } + if err := f.saveSchema(); err != nil { + f.Close() + return nil, err + } + // Add to index's frame lookup. i.frames[name] = f diff --git a/index_test.go b/index_test.go index bf59236c6..1d8a0330b 100644 --- a/index_test.go +++ b/index_test.go @@ -17,6 +17,7 @@ package pilosa_test import ( "io/ioutil" "os" + "reflect" "testing" "github.com/pilosa/pilosa" @@ -88,6 +89,123 @@ func TestIndex_CreateFrame(t *testing.T) { }) }) + // Ensure frame can include range columns. + t.Run("RangeEnabled", func(t *testing.T) { + t.Run("OK", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create frame with schema and verify it exists. + if f, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, + }, + }); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, + }, + }) { + t.Fatalf("unexpected schema: %#v", f.Schema()) + } + + // Reopen the index & verify the fields are loaded. + if err := index.Reopen(); err != nil { + t.Fatal(err) + } else if f := index.Frame("f"); !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, + }, + }) { + t.Fatalf("unexpected schema after reopen: %#v", f.Schema()) + } + }) + + t.Run("ErrInverseRangeNotAllowed", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + InverseEnabled: true, + RangeEnabled: true, + }); err != pilosa.ErrInverseRangeNotAllowed { + t.Fatal(err) + } + }) + + t.Run("ErrRangeCacheNotAllowed", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + CacheType: pilosa.CacheTypeRanked, + }); err != pilosa.ErrRangeCacheNotAllowed { + t.Fatal(err) + } + }) + + t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt}, + }, + }); err != pilosa.ErrFrameFieldsNotAllowed { + t.Fatal(err) + } + }) + + t.Run("ErrFieldNameRequired", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "", Type: pilosa.FieldTypeInt}, + }, + }); err != pilosa.ErrFieldNameRequired { + t.Fatal(err) + } + }) + + t.Run("ErrInvalidFieldType", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: "bad_type"}, + }, + }); err != pilosa.ErrInvalidFieldType { + t.Fatal(err) + } + }) + + t.Run("ErrInvalidFieldRange", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, + }, + }); err != pilosa.ErrInvalidFieldRange { + t.Fatal(err) + } + }) + }) + // Ensure frame cannot be created with a matching row label. t.Run("ErrColumnRowLabelEqual", func(t *testing.T) { t.Run("Explicit", func(t *testing.T) { diff --git a/internal/private.pb.go b/internal/private.pb.go index eba920486..a7e942bc5 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -25,6 +25,8 @@ Index NodeStatus ClusterStatus + FrameSchema + Field */ package internal @@ -61,6 +63,7 @@ type FrameMeta struct { CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + RangeEnabled bool `protobuf:"varint,6,opt,name=RangeEnabled,proto3" json:"RangeEnabled,omitempty"` } func (m *FrameMeta) Reset() { *m = FrameMeta{} } @@ -268,6 +271,34 @@ func (m *ClusterStatus) GetNodes() []*NodeStatus { return nil } +type FrameSchema struct { + Fields []*Field `protobuf:"bytes,1,rep,name=Fields" json:"Fields,omitempty"` +} + +func (m *FrameSchema) Reset() { *m = FrameSchema{} } +func (m *FrameSchema) String() string { return proto.CompactTextString(m) } +func (*FrameSchema) ProtoMessage() {} +func (*FrameSchema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } + +func (m *FrameSchema) GetFields() []*Field { + if m != nil { + return m.Fields + } + return nil +} + +type Field struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` + Min int64 `protobuf:"varint,3,opt,name=Min,proto3" json:"Min,omitempty"` + Max int64 `protobuf:"varint,4,opt,name=Max,proto3" json:"Max,omitempty"` +} + +func (m *Field) Reset() { *m = Field{} } +func (m *Field) String() string { return proto.CompactTextString(m) } +func (*Field) ProtoMessage() {} +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } + func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") @@ -285,6 +316,8 @@ func init() { proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") + proto.RegisterType((*FrameSchema)(nil), "internal.FrameSchema") + proto.RegisterType((*Field)(nil), "internal.Field") } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -364,6 +397,16 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) i += copy(dAtA[i:], m.TimeQuantum) } + if m.RangeEnabled { + dAtA[i] = 0x30 + i++ + if m.RangeEnabled { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -899,6 +942,76 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *FrameSchema) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FrameSchema) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Fields) > 0 { + for _, msg := range m.Fields { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *Field) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Field) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Type) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) + i += copy(dAtA[i:], m.Type) + } + if m.Min != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) + } + if m.Max != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) + } + return i, nil +} + func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) dAtA[offset+1] = uint8(v >> 8) @@ -961,6 +1074,9 @@ func (m *FrameMeta) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.RangeEnabled { + n += 2 + } return n } @@ -1193,6 +1309,38 @@ func (m *ClusterStatus) Size() (n int) { return n } +func (m *FrameSchema) Size() (n int) { + var l int + _ = l + if len(m.Fields) > 0 { + for _, e := range m.Fields { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + +func (m *Field) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Type) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Min != 0 { + n += 1 + sovPrivate(uint64(m.Min)) + } + if m.Max != 0 { + n += 1 + sovPrivate(uint64(m.Max)) + } + return n +} + func sovPrivate(x uint64) (n int) { for { n++ @@ -1469,6 +1617,26 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { } m.TimeQuantum = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RangeEnabled", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.RangeEnabled = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3300,6 +3468,233 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } return nil } +func (m *FrameSchema) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FrameSchema: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FrameSchema: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Fields = append(m.Fields, &Field{}) + if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Field) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Field: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Field: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Type = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Min", wireType) + } + m.Min = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Min |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) + } + m.Max = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Max |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -3408,45 +3803,50 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 640 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x4e, 0x14, 0x4d, - 0x10, 0xfe, 0x67, 0x77, 0x96, 0x7f, 0xa7, 0x08, 0x08, 0x2d, 0x31, 0x23, 0x21, 0x9b, 0x4d, 0x1f, - 0x04, 0x39, 0x70, 0xc0, 0x8b, 0x51, 0x0f, 0x86, 0x5d, 0x0c, 0x93, 0x08, 0xc6, 0x5e, 0xe2, 0xd1, - 0xa4, 0x81, 0x8a, 0x4e, 0x98, 0x9d, 0x59, 0xa7, 0x7b, 0x80, 0xf5, 0xe0, 0x73, 0x98, 0x78, 0xf2, - 0x01, 0x7c, 0x0f, 0x8f, 0x3e, 0x82, 0xc1, 0x17, 0x31, 0x5d, 0xdd, 0x33, 0xb3, 0x0e, 0x22, 0xd1, - 0x5b, 0xd7, 0x57, 0xd5, 0xf5, 0x7d, 0xfd, 0x4d, 0xd5, 0xc0, 0xc2, 0x24, 0x8f, 0xcf, 0xa4, 0xc6, - 0xad, 0x49, 0x9e, 0xe9, 0x8c, 0x75, 0xe3, 0x54, 0x63, 0x9e, 0xca, 0x84, 0xbf, 0x80, 0x20, 0x4a, - 0x4f, 0xf0, 0x62, 0x1f, 0xb5, 0x64, 0x7d, 0x98, 0x1f, 0x64, 0x49, 0x31, 0x4e, 0x9f, 0xcb, 0x23, - 0x4c, 0x42, 0xaf, 0xef, 0x6d, 0x04, 0x62, 0x16, 0x32, 0x15, 0x87, 0xf1, 0x18, 0x5f, 0x16, 0x32, - 0xd5, 0xc5, 0x38, 0x6c, 0xd9, 0x8a, 0x19, 0x88, 0x7f, 0xf1, 0x20, 0x78, 0x96, 0xcb, 0x31, 0x52, - 0xc7, 0x55, 0xe8, 0x8a, 0xec, 0x7c, 0xb6, 0x5d, 0x15, 0xb3, 0x7b, 0xb0, 0x18, 0xa5, 0x67, 0x98, - 0x2b, 0xdc, 0x4d, 0xe5, 0x51, 0x82, 0x27, 0xd4, 0xae, 0x2b, 0x1a, 0x28, 0x5b, 0x83, 0x60, 0x20, - 0x8f, 0xdf, 0xe2, 0xe1, 0x74, 0x82, 0x61, 0x9b, 0x9a, 0xd4, 0x40, 0x95, 0x1d, 0xc5, 0xef, 0x31, - 0xf4, 0xfb, 0xde, 0xc6, 0x82, 0xa8, 0x81, 0xa6, 0xde, 0xce, 0x55, 0xbd, 0x1c, 0x16, 0xa3, 0xf1, - 0x24, 0xcb, 0xb5, 0x40, 0x35, 0xc9, 0x52, 0x85, 0x6c, 0x09, 0xda, 0xbb, 0x79, 0xee, 0xe4, 0x9a, - 0x23, 0xff, 0x00, 0x4b, 0x3b, 0x49, 0x76, 0x7c, 0x3a, 0x94, 0x5a, 0x0a, 0x7c, 0x57, 0xa0, 0xd2, - 0x6c, 0x05, 0x3a, 0x64, 0x9c, 0xab, 0xb3, 0x81, 0x41, 0xe9, 0xf1, 0xce, 0x19, 0x1b, 0x18, 0x94, - 0xee, 0x93, 0x7a, 0x5f, 0xd8, 0xc0, 0xa0, 0xa3, 0x24, 0x3e, 0xb6, 0xaa, 0x7d, 0x61, 0x03, 0xc6, - 0xc0, 0x7f, 0x15, 0xe3, 0xb9, 0x93, 0x4a, 0x67, 0x1e, 0xc1, 0xf2, 0x0c, 0xbf, 0x93, 0x79, 0x07, - 0xe6, 0x44, 0x76, 0x1e, 0x0d, 0x55, 0xe8, 0xf5, 0xdb, 0x1b, 0xbe, 0x70, 0x11, 0x19, 0x42, 0x5f, - 0xcc, 0xa4, 0x5a, 0x94, 0xaa, 0x01, 0x7e, 0x17, 0x3a, 0xe4, 0x8e, 0x79, 0x65, 0x7d, 0xd7, 0x1c, - 0xf9, 0x27, 0x0f, 0x96, 0xf7, 0xe5, 0x05, 0xc9, 0x50, 0x15, 0xcd, 0x1e, 0x04, 0x15, 0x48, 0xd5, - 0xf3, 0xdb, 0x9b, 0x5b, 0xe5, 0xf8, 0x6c, 0x5d, 0xa9, 0xaf, 0x91, 0xdd, 0x54, 0xe7, 0x53, 0x51, - 0x5f, 0x5e, 0x7d, 0x02, 0x8b, 0xbf, 0x26, 0x8d, 0x86, 0x53, 0x9c, 0x96, 0x4e, 0x9f, 0xe2, 0xd4, - 0x78, 0x72, 0x26, 0x93, 0xc2, 0xfa, 0xe7, 0x0b, 0x1b, 0x3c, 0x6a, 0x3d, 0xf4, 0xf8, 0x6b, 0x60, - 0x83, 0x1c, 0xa5, 0x46, 0x6a, 0xb0, 0x8f, 0x4a, 0xc9, 0x37, 0x78, 0xfd, 0x57, 0xb0, 0xce, 0xb6, - 0x66, 0x9d, 0x5d, 0x83, 0x20, 0x52, 0x6e, 0xb6, 0xe8, 0x4b, 0x74, 0x45, 0x0d, 0xf0, 0x4d, 0x60, - 0x43, 0x4c, 0x50, 0xa3, 0x5b, 0x87, 0x3f, 0xf4, 0xe7, 0xa3, 0x52, 0xcb, 0xcd, 0xb5, 0x6c, 0x1d, - 0x7c, 0xb3, 0x09, 0x24, 0x65, 0x7e, 0xfb, 0x76, 0x6d, 0x5d, 0xb5, 0x76, 0x82, 0x0a, 0x78, 0x5c, - 0x36, 0x75, 0xdb, 0x73, 0xc3, 0x03, 0x7f, 0x33, 0x66, 0x25, 0x55, 0xbb, 0x49, 0x55, 0xed, 0xa3, - 0xa3, 0x7a, 0x5a, 0xbe, 0xf5, 0x5f, 0xa9, 0xf8, 0xd0, 0xa1, 0x66, 0x5c, 0x0f, 0x4c, 0xd6, 0xde, - 0xa1, 0xf3, 0xf5, 0x4f, 0x6e, 0xea, 0xf8, 0xec, 0x39, 0xca, 0xbf, 0x6b, 0xd3, 0x70, 0xce, 0xfc, - 0x64, 0xca, 0xc1, 0x72, 0x1b, 0x56, 0xc5, 0x6c, 0x1d, 0xe6, 0x88, 0x55, 0x85, 0x3e, 0xcd, 0xee, - 0xad, 0x86, 0x1a, 0xe1, 0xd2, 0x66, 0x9d, 0xdc, 0x90, 0x77, 0xec, 0x3a, 0xd9, 0x88, 0x4b, 0x80, - 0x83, 0xec, 0x04, 0x47, 0x5a, 0xea, 0x42, 0x19, 0x9d, 0x7b, 0x99, 0xd2, 0xa5, 0x4e, 0x73, 0xa6, - 0x69, 0xd3, 0x52, 0x57, 0x0e, 0x51, 0xc0, 0xee, 0xc3, 0xff, 0xa4, 0x13, 0x55, 0xd8, 0x6e, 0x32, - 0x53, 0x42, 0x94, 0x79, 0xfe, 0x18, 0x16, 0x06, 0x49, 0xa1, 0x34, 0xe6, 0x8e, 0x65, 0x13, 0x3a, - 0x86, 0xb3, 0xdc, 0xb7, 0x95, 0xfa, 0x66, 0x2d, 0x45, 0xd8, 0x92, 0x9d, 0xa5, 0xaf, 0x97, 0x3d, - 0xef, 0xdb, 0x65, 0xcf, 0xfb, 0x7e, 0xd9, 0xf3, 0x3e, 0xfe, 0xe8, 0xfd, 0x77, 0x34, 0x47, 0xff, - 0xf8, 0x07, 0x3f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x5a, 0x1c, 0x79, 0x08, 0xf4, 0x05, 0x00, 0x00, + // 709 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xc1, 0x6e, 0x13, 0x49, + 0x10, 0xdd, 0xb1, 0xc7, 0x5e, 0xbb, 0xbc, 0xc9, 0x26, 0xbd, 0xd1, 0xca, 0x44, 0x91, 0x65, 0xf5, + 0x81, 0x84, 0x1c, 0x72, 0x08, 0x12, 0x42, 0xc0, 0x01, 0xc5, 0x0e, 0x8a, 0x25, 0x1c, 0x44, 0x3b, + 0xe2, 0x88, 0xd4, 0xb1, 0x4b, 0xc9, 0x28, 0xe3, 0x19, 0x33, 0xdd, 0x4e, 0x6c, 0x0e, 0x7c, 0x07, + 0x12, 0x27, 0xfe, 0x86, 0x23, 0xfc, 0x01, 0x0a, 0x3f, 0x82, 0xba, 0xba, 0x67, 0xc6, 0x99, 0x24, + 0x44, 0x70, 0xab, 0x7a, 0x55, 0x5d, 0xf5, 0xea, 0xb9, 0x6a, 0x0c, 0x4b, 0x93, 0x24, 0x38, 0x97, + 0x1a, 0x77, 0x26, 0x49, 0xac, 0x63, 0x56, 0x0b, 0x22, 0x8d, 0x49, 0x24, 0x43, 0xfe, 0x0a, 0xea, + 0xbd, 0x68, 0x84, 0xb3, 0x3e, 0x6a, 0xc9, 0xda, 0xd0, 0xe8, 0xc4, 0xe1, 0x74, 0x1c, 0xbd, 0x94, + 0xc7, 0x18, 0x36, 0xbd, 0xb6, 0xb7, 0x55, 0x17, 0x8b, 0x90, 0xc9, 0x38, 0x0a, 0xc6, 0xf8, 0x7a, + 0x2a, 0x23, 0x3d, 0x1d, 0x37, 0x4b, 0x36, 0x63, 0x01, 0xe2, 0xdf, 0x3c, 0xa8, 0xbf, 0x48, 0xe4, + 0x18, 0xa9, 0xe2, 0x3a, 0xd4, 0x44, 0x7c, 0xb1, 0x58, 0x2e, 0xf3, 0xd9, 0x7d, 0x58, 0xee, 0x45, + 0xe7, 0x98, 0x28, 0xdc, 0x8f, 0xe4, 0x71, 0x88, 0x23, 0x2a, 0x57, 0x13, 0x05, 0x94, 0x6d, 0x40, + 0xbd, 0x23, 0x87, 0xa7, 0x78, 0x34, 0x9f, 0x60, 0xb3, 0x4c, 0x45, 0x72, 0x20, 0x8b, 0x0e, 0x82, + 0xf7, 0xd8, 0xf4, 0xdb, 0xde, 0xd6, 0x92, 0xc8, 0x81, 0x22, 0xdf, 0xca, 0x35, 0xbe, 0x8c, 0xc3, + 0x3f, 0x42, 0x46, 0x27, 0x19, 0x87, 0x2a, 0x71, 0xb8, 0x82, 0x71, 0x0e, 0xcb, 0xbd, 0xf1, 0x24, + 0x4e, 0xb4, 0x40, 0x35, 0x89, 0x23, 0x85, 0x6c, 0x05, 0xca, 0xfb, 0x49, 0xe2, 0x46, 0x32, 0x26, + 0xff, 0x00, 0x2b, 0x7b, 0x61, 0x3c, 0x3c, 0xeb, 0x4a, 0x2d, 0x05, 0xbe, 0x9b, 0xa2, 0xd2, 0x6c, + 0x0d, 0x2a, 0x24, 0xae, 0xcb, 0xb3, 0x8e, 0x41, 0x49, 0x20, 0xa7, 0x9e, 0x75, 0x0c, 0x4a, 0xef, + 0x69, 0x42, 0x5f, 0x58, 0xc7, 0xa0, 0x83, 0x30, 0x18, 0xda, 0xc9, 0x7c, 0x61, 0x1d, 0xc6, 0xc0, + 0x7f, 0x13, 0xe0, 0x85, 0x1b, 0x87, 0x6c, 0xde, 0x83, 0xd5, 0x85, 0xfe, 0x8e, 0xe6, 0xff, 0x50, + 0x15, 0xf1, 0x45, 0xaf, 0xab, 0x9a, 0x5e, 0xbb, 0xbc, 0xe5, 0x0b, 0xe7, 0x91, 0x68, 0xf4, 0xab, + 0x9a, 0x50, 0x89, 0x42, 0x39, 0xc0, 0xef, 0x41, 0x85, 0x14, 0x34, 0x53, 0xe6, 0x6f, 0x8d, 0xc9, + 0x3f, 0x79, 0xb0, 0xda, 0x97, 0x33, 0xa2, 0xa1, 0xb2, 0x36, 0x07, 0x50, 0xcf, 0x40, 0xca, 0x6e, + 0xec, 0x6e, 0xef, 0xa4, 0x2b, 0xb6, 0x73, 0x2d, 0x3f, 0x47, 0xf6, 0x23, 0x9d, 0xcc, 0x45, 0xfe, + 0x78, 0xfd, 0x19, 0x2c, 0x5f, 0x0d, 0x1a, 0x0e, 0x67, 0x38, 0x4f, 0x95, 0x3e, 0xc3, 0xb9, 0xd1, + 0xe4, 0x5c, 0x86, 0x53, 0xab, 0x9f, 0x2f, 0xac, 0xf3, 0xa4, 0xf4, 0xd8, 0xe3, 0x6f, 0x81, 0x75, + 0x12, 0x94, 0x1a, 0xa9, 0x40, 0x1f, 0x95, 0x92, 0x27, 0x78, 0xfb, 0xaf, 0x60, 0x95, 0x2d, 0x2d, + 0x2a, 0xbb, 0x01, 0xf5, 0x9e, 0x72, 0xfb, 0x47, 0xbf, 0x44, 0x4d, 0xe4, 0x00, 0xdf, 0x06, 0xd6, + 0xc5, 0x10, 0x35, 0xba, 0x93, 0xf9, 0x45, 0x7d, 0x3e, 0x48, 0xb9, 0xdc, 0x9d, 0xcb, 0x36, 0xc1, + 0x37, 0xd7, 0x42, 0x54, 0x1a, 0xbb, 0xff, 0xe5, 0xd2, 0x65, 0xa7, 0x29, 0x28, 0x81, 0x07, 0x69, + 0x51, 0x77, 0x61, 0x77, 0x0c, 0x78, 0xc3, 0x9a, 0xa5, 0xad, 0xca, 0xc5, 0x56, 0xd9, 0xcd, 0xba, + 0x56, 0xcf, 0xd3, 0x59, 0xff, 0xb4, 0x15, 0xef, 0x3a, 0xd4, 0xac, 0xeb, 0xa1, 0x89, 0xda, 0x37, + 0x64, 0xdf, 0x3e, 0x72, 0x91, 0xc7, 0x67, 0xcf, 0xb5, 0xfc, 0xbd, 0x32, 0x05, 0xe5, 0xcc, 0x87, + 0x28, 0x5d, 0x2c, 0x77, 0x61, 0x99, 0xcf, 0x36, 0xa1, 0x4a, 0x5d, 0x55, 0xd3, 0xa7, 0xdd, 0xfd, + 0xb7, 0xc0, 0x46, 0xb8, 0xb0, 0x39, 0x27, 0xb7, 0xe4, 0x15, 0x7b, 0x4e, 0xd6, 0xe3, 0x12, 0xe0, + 0x30, 0x1e, 0xe1, 0x40, 0x4b, 0x3d, 0x55, 0x86, 0xe7, 0x41, 0xac, 0x74, 0xca, 0xd3, 0xd8, 0xb4, + 0x6d, 0x5a, 0xea, 0x4c, 0x21, 0x72, 0xd8, 0x03, 0xf8, 0x9b, 0x78, 0xa2, 0x6a, 0x96, 0x8b, 0x9d, + 0x29, 0x20, 0xd2, 0x38, 0x7f, 0x0a, 0x4b, 0x9d, 0x70, 0xaa, 0x34, 0x26, 0xae, 0xcb, 0x36, 0x54, + 0x4c, 0xcf, 0xf4, 0xde, 0xd6, 0xf2, 0x97, 0x39, 0x15, 0x61, 0x53, 0xf8, 0x23, 0x68, 0xd0, 0x04, + 0x83, 0xe1, 0x29, 0x8e, 0x25, 0xcd, 0x1b, 0x60, 0x38, 0x4a, 0xdf, 0x2e, 0xce, 0x6b, 0x70, 0xe1, + 0xc2, 0x7c, 0x00, 0x15, 0xb2, 0x6e, 0x94, 0x9e, 0x81, 0x4f, 0x5f, 0x64, 0x3b, 0x11, 0xd9, 0xe6, + 0x58, 0xfb, 0x41, 0x44, 0x02, 0x97, 0x85, 0x31, 0x09, 0x91, 0x33, 0xfa, 0x7c, 0x19, 0x44, 0xce, + 0xf6, 0x56, 0xbe, 0x5c, 0xb6, 0xbc, 0xaf, 0x97, 0x2d, 0xef, 0xfb, 0x65, 0xcb, 0xfb, 0xf8, 0xa3, + 0xf5, 0xd7, 0x71, 0x95, 0xfe, 0x94, 0x1e, 0xfe, 0x0c, 0x00, 0x00, 0xff, 0xff, 0x5d, 0x3a, 0x0c, + 0x81, 0xa5, 0x06, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index c7c792f4a..be2967880 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -13,6 +13,7 @@ message FrameMeta { string CacheType = 3; uint32 CacheSize = 4; string TimeQuantum = 5; + bool RangeEnabled = 6; } message ImportResponse { @@ -88,3 +89,14 @@ message NodeStatus { message ClusterStatus { repeated NodeStatus Nodes = 1; } + +message FrameSchema { + repeated Field Fields = 1; +} + +message Field { + string Name = 1; + string Type = 2; + int64 Min = 3; + int64 Max = 4; +} diff --git a/internal/public.pb.go b/internal/public.pb.go index 9eb005c17..75d79133f 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -2576,7 +2576,7 @@ func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ // 576 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40, 0x10, 0xa5, 0x63, 0xe7, 0x57, 0xf9, 0x28, 0x6a, 0xf1, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, 0x57, 0x19, 0x69, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xa6, 0x33, 0xb0, 0xf7, 0xcc, 0xb4, 0x06, 0x4b, 0xfe, 0xd1, 0xdd, 0x16, 0xe4, 0x00, 0xec, 0x91, 0xd8, 0x70, 0x03, 0x38, 0x0a, 0x4b, diff --git a/pilosa.go b/pilosa.go index 13ade19ea..978ebaf30 100644 --- a/pilosa.go +++ b/pilosa.go @@ -36,6 +36,13 @@ var ( ErrFrameInverseDisabled = errors.New("frame inverse disabled") ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") + ErrFieldNameRequired = errors.New("field name required") + ErrInvalidFieldType = errors.New("invalid field type") + ErrInvalidFieldRange = errors.New("invalid field range") + ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") + ErrRangeCacheNotAllowed = errors.New("range cache not allowed") + ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") + ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type")