From 47e83c31c6cc2b4190c68ffc7253a013906c78f5 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 1 Jun 2017 15:50:30 -0500 Subject: [PATCH 01/20] capture runtime memory metrics in StatsD --- server.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server.go b/server.go index 66034e5d6..ef2c09f76 100644 --- a/server.go +++ b/server.go @@ -460,6 +460,7 @@ func (s *Server) monitorRuntime() { return } + var m runtime.MemStats ticker := time.NewTicker(s.MetricInterval) defer ticker.Stop() @@ -481,6 +482,14 @@ func (s *Server) monitorRuntime() { // Record the number of go routines s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) + + // Runtime memory metrics + runtime.ReadMemStats(&m) + s.Holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0) + s.Holder.Stats.Gauge("HeapInuse", float64(m.HeapInuse), 1.0) + s.Holder.Stats.Gauge("StackInuse", float64(m.StackInuse), 1.0) + s.Holder.Stats.Gauge("Mallocs", float64(m.Mallocs), 1.0) + s.Holder.Stats.Gauge("Frees", float64(m.Frees), 1.0) } } From 6b307f32e0b933b93fa3da650a71a6b36785f766 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 7 Jun 2017 15:03:05 -0500 Subject: [PATCH 02/20] make sure glide is installed before `glide install` If glide is not installed, then $(GLIDE) is empty, so I guess the $(GLIDE) target gets ignored? The impetus for this is that `make install` was failing on a fresh ubuntu instance saying glide wasn't found. --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 563059137..ca672df1c 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,9 @@ $(GLIDE): make glide vendor: $(GLIDE) glide.yaml +ifndef GLIDE + curl https://glide.sh/get | sh +endif glide install glide.lock: glide glide.yaml From f8e0f123038d634995c8385b357bb6539a79d9df Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 8 Jun 2017 15:53:54 -0500 Subject: [PATCH 03/20] Pilosa 0.4.0 docs updates and Homebrew installation guide --- docs/api-reference.md | 2 +- docs/installation.md | 184 +++++++++++++++++++++++++++--------------- 2 files changed, 122 insertions(+), 64 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 55bbde24e..39f66683f 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -240,6 +240,6 @@ curl -XGET localhost:10101/version Response: ``` -{"version":"v0.3.0-353-ge633247"} +{"version":"v0.4.0"} ``` diff --git a/docs/installation.md b/docs/installation.md index c97ed4303..8ee8010d7 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -9,25 +9,79 @@ Pilosa is currently available for [MacOS](#installing-on-macos) and [Linux](#ins ### Installing on MacOS -There are three ways to install Pilosa on MacOS: download the binary (recommended), build from source, or use Docker. +There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) (recommended), download the binary, build from source, or use Docker. + +#### Use Homebrew + +1. Update your Homebrew formulas: + ``` + brew update + ``` + +2. Install Pilosa + ``` + brew install pilosa + ``` + +3. Make sure Pilosa is installed successfully: + ``` + pilosa + ``` + + If you see something like: + ``` + Pilosa is a fast index to turbocharge your database. + + This binary contains Pilosa itself, as well as common + tools for administering pilosa, importing/exporting data, + backing up, and more. Complete documentation is available + at https://www.pilosa.com/docs/ + + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 + + Usage: + pilosa [command] + + Available Commands: + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. + + Flags: + -c, --config string Configuration file to read from. + + Use "pilosa [command] --help" for more information about a command. + ``` + + You're good to go! #### Download the Binary 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.3.1/pilosa-v0.3.1-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.4.0/pilosa-v0.4.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.3.1-darwin-amd64.tar.gz + tar xfz pilosa-v0.4.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.3.1-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.4.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -42,26 +96,27 @@ There are three ways to install Pilosa on MacOS: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. - sort Sort import data for optimal import performance. + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. Flags: -c, --config string Configuration file to read from. @@ -102,26 +157,27 @@ There are three ways to install Pilosa on MacOS: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. - sort Sort import data for optimal import performance. + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. Flags: -c, --config string Configuration file to read from. @@ -165,19 +221,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.3.1/pilosa-v0.3.1-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.4.0/pilosa-v0.4.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.3.1-linux-amd64.tar.gz + tar xfz pilosa-v0.4.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.3.1-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.4.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -192,26 +248,27 @@ There are three ways to install Pilosa on Linux: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. - sort Sort import data for optimal import performance. + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. Flags: -c, --config string Configuration file to read from. @@ -252,26 +309,27 @@ There are three ways to install Pilosa on Linux: download the binary (recommende This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available - at http://pilosa.com/docs + at https://www.pilosa.com/docs/ - Version: v0.3.0-279-gcf7082f - Build Time: 2017-04-21T15:36:08+0000 + Version: v0.4.0 + Build Time: 2017-06-08T19:44:21+0000 Usage: pilosa [command] Available Commands: - backup Backup data from pilosa. - bench Benchmark operations. - check Do a consistency check on a pilosa data file. - config Print the default configuration. - export Export data from pilosa. - help Help about any command - import Bulk load data into pilosa. - inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. - server Run Pilosa. - sort Sort import data for optimal import performance. + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the current configuration. + export Export data from pilosa. + generate-config Print the default configuration. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. Flags: -c, --config string Configuration file to read from. From 472e9bd2e924ee4831b026db1b9e9b7021da9ffb Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 9 Jun 2017 12:01:10 -0500 Subject: [PATCH 04/20] Add PULL_REQUEST_TEMPLATE file. --- .github/PULL_REQUEST_TEMPLATE.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..d96b88aeb --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,22 @@ +## Overview + +[Describe what this pull request addresses.] + +Fixes # + +## Pull request checklist + +- [ ] I have updated the [documentation](https://github.com/pilosa/pilosa/tree/master/docs). +- [ ] I have resolved any merge conflicts. +- [ ] I have included tests that cover my changes. +- [ ] All new and existing tests pass. + +## Code review checklist +This is the checklist that the reviewer will follow while reviewing your pull request. You do not need to do anything with this checklist, but be aware of what the reviewer will be looking for. + +- [ ] Ensure that any changes to external docs have been included in this pull request. +- [ ] If the changes require that minor/major versions need to be updated, tag the PR appropriately. +- [ ] Ensure the new code is [properly commented](https://github.com/golang/go/wiki/CodeReviewComments#doc-comments) and follows [Idiomatic Go](https://dmitri.shuralyov.com/idiomatic-go). +- [ ] Check that tests have been written and that they cover the new functionality. +- [ ] Run tests and ensure they pass. +- [ ] Build and run the code, performing any applicable integration testing. From 571d249ea12d86b9e9f666e4f5f589cab377ff3e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 9 Jun 2017 14:35:57 -0500 Subject: [PATCH 05/20] Add contributing guide and CLA info to pull request template --- .github/PULL_REQUEST_TEMPLATE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d96b88aeb..081a20073 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,6 +6,8 @@ Fixes # ## Pull request checklist +- [ ] I have read the [contributing guide](https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md). +- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa). - [ ] I have updated the [documentation](https://github.com/pilosa/pilosa/tree/master/docs). - [ ] I have resolved any merge conflicts. - [ ] I have included tests that cover my changes. From 1e793a1c524179e1d4503ccda1a29dc95360e10e Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 12 Jun 2017 13:16:19 -0500 Subject: [PATCH 06/20] Count open file handles as a StatsD metric. --- server.go | 31 +++++++++++++++++++++++++++++++ server/server_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/server.go b/server.go index f9719c563..7f2027f10 100644 --- a/server.go +++ b/server.go @@ -24,8 +24,10 @@ import ( "net/http" "net/url" "os" + "os/exec" "runtime" "strconv" + "strings" "sync" "time" @@ -486,6 +488,9 @@ func (s *Server) monitorRuntime() { // Record the number of go routines s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) + // Open File handles + s.Holder.Stats.Gauge("OpenFiles", float64(CountOpenFiles()), 1.0) + // Runtime memory metrics runtime.ReadMemStats(&m) s.Holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0) @@ -496,6 +501,32 @@ func (s *Server) monitorRuntime() { } } +// CountOpenFiles on opperating systems that support lsof +func CountOpenFiles() int { + count := 0 + + switch runtime.GOOS { + case "darwin": + fallthrough + case "linux": + fallthrough + case "unix": + fallthrough + case "freebsd": + out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -p %v", os.Getpid())).Output() + if err != nil { + log.Fatal(err) + } + lines := strings.Split(string(out), "\n") + count = len(lines) - 1 + case "windows": + // TODO: count open file handles on windows + default: + + } + return count +} + // StatusHandler specifies two methods which an object must implement to share // state in the cluster. These are used by the GossipNodeSet to implement the // LocalState and MergeRemoteState methods of memberlist.Delegate diff --git a/server/server_test.go b/server/server_test.go index cd82ba007..f1ee11b23 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -25,7 +25,9 @@ import ( "net" "net/http" "os" + "path/filepath" "reflect" + "runtime" "sort" "strconv" "strings" @@ -372,6 +374,42 @@ path = "/path/to/plugins" } } +// tempMkdir makes a temporary directory +func tempMkdir(t *testing.T) string { + dir, err := ioutil.TempDir("", "pilosatemp") + if err != nil { + t.Fatalf("failed to create test directory: %s", err) + } + return dir +} + +// Ensure the file handle count is working +func TestCountOpenFiles(t *testing.T) { + // Windows is not supported yet + supported := []string{"darwin", "linux", "unix", "freebsd"} + sort.Strings(supported) + i := sort.Search(len(supported), + func(i int) bool { return supported[i] >= runtime.GOOS }) + if i == len(supported) { + return + } + + // Create directory store temp file + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + count := pilosa.CountOpenFiles() + testFile := filepath.Join(testDir, "test.txt") + _, err := os.Create(testFile) + if err != nil { + t.Fatalf("create test file failed: %s", err) + } + + if pilosa.CountOpenFiles() < count+1 { + t.Error("Invalid open file handle count") + } +} + // Ensure program can send/receive broadcast messages. func TestMain_SendReceiveMessage(t *testing.T) { m0 := MustRunMain() From 3a43a862ec19e0e75d533d4c9b8110679d28fa78 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sat, 10 Jun 2017 10:31:36 -0600 Subject: [PATCH 07/20] Add Holder test coverage. --- holder_test.go | 235 +++++++++++++++++++++++++++++++++++++++++++++++++ view.go | 2 +- 2 files changed, 236 insertions(+), 1 deletion(-) diff --git a/holder_test.go b/holder_test.go index 52288670a..b6864ffc9 100644 --- a/holder_test.go +++ b/holder_test.go @@ -19,13 +19,231 @@ import ( "context" "io/ioutil" "os" + "path/filepath" "reflect" + "strings" "testing" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" ) +func TestHolder_Open(t *testing.T) { + t.Run("ErrIndexName", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err != nil { + t.Fatal(err) + } else if logOutput := h.LogOutput.String(); !strings.Contains(logOutput, `ERROR opening index: !`) { + t.Fatalf("expected log error:\n%s", logOutput) + } + }) + + t.Run("ErrIndexPermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Chmod(h.IndexPath("test"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(h.IndexPath("test"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrIndexMetaCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".meta"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "unexpected EOF") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".data"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=test, err=invalid database") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFramePermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrFrameMetaCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".meta"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=open frame: name=bar, err=unexpected EOF") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrFrameAttrStoreCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".data"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=open frame: name=bar, err=invalid database") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrViewPermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrViewFragmentsMkdir", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFragmentStoragePermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if _, err := view.SetBit(0, 0); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0666) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if _, err := view.SetBit(0, 0); err != nil { + t.Fatal(err) + } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 2); err != nil { + t.Fatal(err) + } + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: slice=0, err=unmarshal storage") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFragmentCachePermission", func(t *testing.T) { + h := MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if _, err := view.SetBit(0, 0); err != nil { + t.Fatal(err) + } else if err := view.Fragment(0).FlushCache(); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) +} + // Ensure holder can delete an index and its underlying files. func TestHolder_DeleteIndex(t *testing.T) { hldr := MustOpenHolder() @@ -211,6 +429,23 @@ func (h *Holder) Close() error { return h.Holder.Close() } +// Reopen closes the holder and instantiates and opens a new holder. +func (h *Holder) Reopen() error { + if err := h.Holder.Close(); err != nil { + return err + } + + path, logOutput := h.Path, h.Holder.LogOutput + h.Holder = pilosa.NewHolder() + h.Holder.Path = path + h.Holder.LogOutput = logOutput + if err := h.Holder.Open(); err != nil { + return err + } + + return nil +} + // MustCreateIndexIfNotExists returns a given index. Panic on error. func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOptions) *Index { idx, err := h.Holder.CreateIndexIfNotExists(index, opt) diff --git a/view.go b/view.go index 5c6198a5e..367a69b5a 100644 --- a/view.go +++ b/view.go @@ -144,7 +144,7 @@ func (v *View) openFragments() error { frag := v.newFragment(v.FragmentPath(slice), slice) if err := frag.Open(); err != nil { - return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err) + return fmt.Errorf("open fragment: slice=%d, err=%s", frag.Slice(), err) } frag.RowAttrStore = v.RowAttrStore v.fragments[frag.Slice()] = frag From 188eaefb575ae8601ea5999a54c7cd93d63bfbf0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 12 Jun 2017 14:21:32 -0500 Subject: [PATCH 08/20] Remove unintentional dash from LDFLAGS --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ca672df1c..a5113c2ab 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa BUILD_TIME=`date -u +%FT%T%z` -LDFLAGS=-"-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" +LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" default: test pilosa From 76357374879c3e4080a9c1fd53c98ab0c689c970 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 12 Jun 2017 15:20:24 -0500 Subject: [PATCH 09/20] Add coverage tools to Makefile `make cover-viz` will now generate an HTML coverage report for the entire codebase. --- Makefile | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a5113c2ab..6ef21fda0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release test +.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release test cover cover-pkg cover-viz GLIDE := $(shell command -v glide 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) @@ -6,6 +6,7 @@ PROTOC := $(shell command -v protoc 2>/dev/null) VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa +PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) BUILD_TIME=`date -u +%FT%T%z` LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" @@ -34,7 +35,23 @@ glide.lock: glide glide.yaml vendor-update: glide.lock test: vendor - go test $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) $(TESTFLAGS) + go test $(PKGS) $(TESTFLAGS) + +cover: vendor + mkdir -p build/coverage + echo "mode: set" > build/coverage/all.out + for pkg in $(PKGS) ; do \ + make cover-pkg PKG=$$pkg ; \ + done + +cover-pkg: + mkdir -p build/coverage + touch build/coverage/$(subst /,-,$(PKG)).out + go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG) + tail +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out + +cover-viz: cover + go tool cover -html=build/coverage/all.out pilosa: vendor go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa From cd532e08cbc188fc089a9e722b25c2a187fe9b12 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 13 Jun 2017 16:14:45 -0500 Subject: [PATCH 10/20] Add -b option to lsof to avoid kernel blocks. --- docs/administration.md | 2 ++ server.go | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 322b68e12..e562f94ce 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -157,3 +157,5 @@ We currently track the following events Garbage Collection: Event count when Garbage Collection occurs. Goroutines: Number of running Goroutines. + +OpenFiles: Number of open file handles associated with running Pilosa process ID. \ No newline at end of file diff --git a/server.go b/server.go index 7f2027f10..57f3291eb 100644 --- a/server.go +++ b/server.go @@ -513,12 +513,15 @@ func CountOpenFiles() int { case "unix": fallthrough case "freebsd": - out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -p %v", os.Getpid())).Output() + // -b option avoid kernel blocks + pid := os.Getpid() + out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -b -p %v", pid)).Output() if err != nil { log.Fatal(err) } - lines := strings.Split(string(out), "\n") - count = len(lines) - 1 + // only count lines with our pid, avoiding warning messages from -b + lines := strings.Split(string(out), strconv.Itoa(pid)) + count = len(lines) case "windows": // TODO: count open file handles on windows default: From 87f294023319fddebcc7ccdba6525dcc33c762e0 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 14 Jun 2017 09:26:13 -0500 Subject: [PATCH 11/20] declare multiple values in the same switch case --- server.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/server.go b/server.go index 57f3291eb..37fadedd5 100644 --- a/server.go +++ b/server.go @@ -506,13 +506,7 @@ func CountOpenFiles() int { count := 0 switch runtime.GOOS { - case "darwin": - fallthrough - case "linux": - fallthrough - case "unix": - fallthrough - case "freebsd": + case "darwin", "linux", "unix", "freebsd": // -b option avoid kernel blocks pid := os.Getpid() out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -b -p %v", pid)).Output() From 23e7e42d99354e1b310673de00f167228d7572bd Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 19 Jun 2017 13:18:20 -0500 Subject: [PATCH 12/20] Test create input definition across 2 servers --- server/server_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/server/server_test.go b/server/server_test.go index cd82ba007..a92da70a8 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -494,6 +494,29 @@ func TestMain_SendReceiveMessage(t *testing.T) { if maxSlices1["i"] != 2 { t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) } + + // Write input definition to the first node. + if _, err := m0.CreateDefinition("i", "test", `{ + "frames": [{"name": "event-time", + "options": { + "cacheType": "ranked", + "timeQuantum": "YMD" + }}], + "fields": [{"name": "id", + "primaryKey": true + }]} + `); err != nil { + t.Fatal(err) + } + + frame0 := m0.Server.Holder.Frame("i", "event-time") + if frame0 == nil { + t.Fatal("frame not found") + } + frame1 := m1.Server.Holder.Frame("i", "event-time") + if frame1 == nil { + t.Fatal("frame not found") + } } // availablePorts returns a slice of ports that can be used for testing. @@ -615,6 +638,15 @@ func (m *Main) Query(index, rawQuery, query string) (string, error) { return resp.Body, nil } +// CreateDefinition. +func (m *Main) CreateDefinition(index, def, query string) (string, error) { + resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + return resp.Body, nil +} + // SetCommand represents a command to set a bit. type SetCommand struct { ID uint64 From e0bf37da40e7f08921a7cd61e814f44020cfbc86 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 20 Jun 2017 21:37:27 -0500 Subject: [PATCH 13/20] check same rowID for same frame in actions --- input_definition.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/input_definition.go b/input_definition.go index cb15732ca..f05ccb64a 100644 --- a/input_definition.go +++ b/input_definition.go @@ -94,19 +94,19 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { } numPrimaryKey := 0 - countRowID := make(map[uint64]bool) + countRowID := make(map[string]uint64) for _, field := range pb.Fields { var actions []Action for _, action := range field.Actions { if err := i.ValidateAction(action); err != nil { return err } - if action.RowID != 0 { - _, ok := countRowID[action.RowID] - if !ok { - countRowID[action.RowID] = true - } else { + if action.RowID != 0 && action.Frame != ""{ + val, ok := countRowID[action.Frame] + if ok && val == action.RowID { return fmt.Errorf("duplicate rowID with other field: %s", action.RowID) + } else { + countRowID[action.Frame] = action.RowID } } actions = append(actions, Action{ From 0107ddffea943180847d8ff7bbfdbdc97b9059dd Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 20 Jun 2017 21:52:29 -0600 Subject: [PATCH 14/20] BSI range-encoding schema support. --- frame.go | 164 +++++++++++++- index.go | 33 +++ index_test.go | 118 ++++++++++ internal/private.pb.go | 482 +++++++++++++++++++++++++++++++++++++---- internal/private.proto | 12 + internal/public.pb.go | 2 +- pilosa.go | 7 + 7 files changed, 773 insertions(+), 45 deletions(-) diff --git a/frame.go b/frame.go index dbc26541e..17abb5f44 100644 --- a/frame.go +++ b/frame.go @@ -34,6 +34,7 @@ const ( DefaultRowLabel = "rowID" DefaultCacheType = CacheTypeRanked DefaultInverseEnabled = false + DefaultRangeEnabled = false // Default ranked frame cache DefaultCacheSize = 50000 @@ -46,6 +47,7 @@ type Frame struct { index string name string timeQuantum TimeQuantum + schema *FrameSchema views map[string]*View @@ -59,6 +61,7 @@ type Frame struct { rowLabel string cacheType string inverseEnabled bool + rangeEnabled bool // Cache size for ranked frames cacheSize uint32 @@ -74,9 +77,10 @@ func NewFrame(path, index, name string) (*Frame, error) { } return &Frame{ - path: path, - index: index, - name: name, + path: path, + index: index, + name: name, + schema: &FrameSchema{}, views: make(map[string]*View), rowAttrStore: NewAttrStore(filepath.Join(path, ".data")), @@ -86,6 +90,7 @@ func NewFrame(path, index, name string) (*Frame, error) { rowLabel: DefaultRowLabel, inverseEnabled: DefaultInverseEnabled, + rangeEnabled: DefaultRangeEnabled, cacheType: DefaultCacheType, cacheSize: DefaultCacheSize, @@ -172,6 +177,11 @@ func (f *Frame) InverseEnabled() bool { return f.inverseEnabled } +// RangeEnabled returns true if range fields can be stored on this frame. +func (f *Frame) RangeEnabled() bool { + return f.rangeEnabled +} + // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 func (f *Frame) SetCacheSize(v uint32) error { @@ -206,6 +216,7 @@ func (f *Frame) Options() FrameOptions { opt := FrameOptions{ RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, + RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: f.timeQuantum, @@ -224,6 +235,8 @@ func (f *Frame) Open() error { if err := f.loadMeta(); err != nil { return err + } else if err := f.loadSchema(); err != nil { + return err } if err := f.openViews(); err != nil { @@ -286,6 +299,7 @@ func (f *Frame) loadMeta() error { f.rowLabel = DefaultRowLabel f.cacheType = DefaultCacheType f.inverseEnabled = DefaultInverseEnabled + f.rangeEnabled = DefaultRangeEnabled f.cacheSize = DefaultCacheSize return nil } else if err != nil { @@ -300,6 +314,7 @@ func (f *Frame) loadMeta() error { f.timeQuantum = TimeQuantum(pb.TimeQuantum) f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled + f.rangeEnabled = pb.RangeEnabled f.cacheSize = pb.CacheSize // Copy cache type. @@ -317,6 +332,7 @@ func (f *Frame) saveMeta() error { buf, err := proto.Marshal(&internal.FrameMeta{ RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, + RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: string(f.timeQuantum), @@ -333,6 +349,35 @@ func (f *Frame) saveMeta() error { return nil } +// loadSchema reads the schema for the frame. +func (f *Frame) loadSchema() error { + buf, err := ioutil.ReadFile(filepath.Join(f.path, ".schema")) + if os.IsNotExist(err) { + f.schema = &FrameSchema{} + return nil + } else if err != nil { + return err + } + + var pb internal.FrameSchema + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + f.schema = decodeFrameSchema(&pb) + + return nil +} + +// saveSchema writes the current schema to disk. +func (f *Frame) saveSchema() error { + if buf, err := proto.Marshal(encodeFrameSchema(f.schema)); err != nil { + return err + } else if err := ioutil.WriteFile(filepath.Join(f.path, ".schema"), buf, 0666); err != nil { + return err + } + return nil +} + // Close closes the frame and its views. func (f *Frame) Close() error { f.mu.Lock() @@ -352,6 +397,13 @@ func (f *Frame) Close() error { return nil } +// Schema returns the frame's current schema. +func (f *Frame) Schema() *FrameSchema { + f.mu.Lock() + defer f.mu.Unlock() + return f.schema +} + // TimeQuantum returns the time quantum for the frame. func (f *Frame) TimeQuantum() TimeQuantum { f.mu.Lock() @@ -619,6 +671,7 @@ func encodeFrame(f *Frame) *internal.Frame { Meta: &internal.FrameMeta{ RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, + RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: string(f.timeQuantum), @@ -648,9 +701,11 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } type FrameOptions struct { RowLabel string `json:"rowLabel,omitempty"` InverseEnabled bool `json:"inverseEnabled,omitempty"` + RangeEnabled bool `json:"rangeEnabled,omitempty"` CacheType string `json:"cacheType,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + Fields []*Field `json:"fields,omitempty"` } // Encode converts o into its internal representation. @@ -658,12 +713,115 @@ func (o *FrameOptions) Encode() *internal.FrameMeta { return &internal.FrameMeta{ RowLabel: o.RowLabel, InverseEnabled: o.InverseEnabled, + RangeEnabled: o.RangeEnabled, CacheType: o.CacheType, CacheSize: o.CacheSize, TimeQuantum: string(o.TimeQuantum), } } +// FrameSchema represents the list of fields on a frame. +type FrameSchema struct { + Fields []*Field +} + +func encodeFrameSchema(schema *FrameSchema) *internal.FrameSchema { + if schema == nil { + return nil + } + return &internal.FrameSchema{ + Fields: encodeFields(schema.Fields), + } +} + +func decodeFrameSchema(schema *internal.FrameSchema) *FrameSchema { + if schema == nil { + return nil + } + return &FrameSchema{ + Fields: decodeFields(schema.Fields), + } +} + +// List of field data types. +const ( + FieldTypeInt = "int" +) + +func IsValidFieldType(v string) bool { + switch v { + case FieldTypeInt: + return true + default: + return false + } +} + +// Field represents a range field on a frame. +type Field struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Min int `json:"min,omitempty"` + Max int `json:"max,omitempty"` +} + +func ValidateField(f *Field) error { + if f.Name == "" { + return ErrFieldNameRequired + } else if !IsValidFieldType(f.Type) { + return ErrInvalidFieldType + } else if f.Min > f.Max { + return ErrInvalidFieldRange + } + return nil +} + +func encodeFields(a []*Field) []*internal.Field { + if len(a) == 0 { + return nil + } + other := make([]*internal.Field, len(a)) + for i := range a { + other[i] = encodeField(a[i]) + } + return other +} + +func decodeFields(a []*internal.Field) []*Field { + if len(a) == 0 { + return nil + } + other := make([]*Field, len(a)) + for i := range a { + other[i] = decodeField(a[i]) + } + return other +} + +func encodeField(f *Field) *internal.Field { + if f == nil { + return nil + } + return &internal.Field{ + Name: f.Name, + Type: f.Type, + Min: int64(f.Min), + Max: int64(f.Max), + } +} + +func decodeField(f *internal.Field) *Field { + if f == nil { + return nil + } + return &Field{ + Name: f.Name, + Type: f.Type, + Min: int(f.Min), + Max: int(f.Max), + } +} + // importBitSet represents slices of row and column ids. // This is used to sort data during import. type importBitSet struct { diff --git a/index.go b/index.go index 8a1b34aed..a5cdd1911 100644 --- a/index.go +++ b/index.go @@ -387,6 +387,30 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, ErrColumnRowLabelEqual } + // Validate mutually exclusive options if ranges are enabled. + // + // NOTE(https://github.com/pilosa/pilosa/issues/399): + // Cache type should be validated as "none" once it is allowed. + if opt.RangeEnabled { + if opt.InverseEnabled { + return nil, ErrInverseRangeNotAllowed + } else if opt.CacheType != "" && opt.CacheType != CacheTypeLRU { + return nil, ErrRangeCacheNotAllowed + } + opt.CacheSize = 0 + } else { + if len(opt.Fields) > 0 { + return nil, ErrFrameFieldsNotAllowed + } + } + + // Validate fields. + for _, field := range opt.Fields { + if err := ValidateField(field); err != nil { + return nil, err + } + } + // Initialize frame. f, err := i.newFrame(i.FramePath(name), name) if err != nil { @@ -428,6 +452,15 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, err } + // Set schema & save. + f.schema = &FrameSchema{ + Fields: opt.Fields, + } + if err := f.saveSchema(); err != nil { + f.Close() + return nil, err + } + // Add to index's frame lookup. i.frames[name] = f diff --git a/index_test.go b/index_test.go index bf59236c6..1d8a0330b 100644 --- a/index_test.go +++ b/index_test.go @@ -17,6 +17,7 @@ package pilosa_test import ( "io/ioutil" "os" + "reflect" "testing" "github.com/pilosa/pilosa" @@ -88,6 +89,123 @@ func TestIndex_CreateFrame(t *testing.T) { }) }) + // Ensure frame can include range columns. + t.Run("RangeEnabled", func(t *testing.T) { + t.Run("OK", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create frame with schema and verify it exists. + if f, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, + }, + }); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, + }, + }) { + t.Fatalf("unexpected schema: %#v", f.Schema()) + } + + // Reopen the index & verify the fields are loaded. + if err := index.Reopen(); err != nil { + t.Fatal(err) + } else if f := index.Frame("f"); !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, + }, + }) { + t.Fatalf("unexpected schema after reopen: %#v", f.Schema()) + } + }) + + t.Run("ErrInverseRangeNotAllowed", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + InverseEnabled: true, + RangeEnabled: true, + }); err != pilosa.ErrInverseRangeNotAllowed { + t.Fatal(err) + } + }) + + t.Run("ErrRangeCacheNotAllowed", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + CacheType: pilosa.CacheTypeRanked, + }); err != pilosa.ErrRangeCacheNotAllowed { + t.Fatal(err) + } + }) + + t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt}, + }, + }); err != pilosa.ErrFrameFieldsNotAllowed { + t.Fatal(err) + } + }) + + t.Run("ErrFieldNameRequired", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "", Type: pilosa.FieldTypeInt}, + }, + }); err != pilosa.ErrFieldNameRequired { + t.Fatal(err) + } + }) + + t.Run("ErrInvalidFieldType", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: "bad_type"}, + }, + }); err != pilosa.ErrInvalidFieldType { + t.Fatal(err) + } + }) + + t.Run("ErrInvalidFieldRange", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, + }, + }); err != pilosa.ErrInvalidFieldRange { + t.Fatal(err) + } + }) + }) + // Ensure frame cannot be created with a matching row label. t.Run("ErrColumnRowLabelEqual", func(t *testing.T) { t.Run("Explicit", func(t *testing.T) { diff --git a/internal/private.pb.go b/internal/private.pb.go index eba920486..a7e942bc5 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -25,6 +25,8 @@ Index NodeStatus ClusterStatus + FrameSchema + Field */ package internal @@ -61,6 +63,7 @@ type FrameMeta struct { CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + RangeEnabled bool `protobuf:"varint,6,opt,name=RangeEnabled,proto3" json:"RangeEnabled,omitempty"` } func (m *FrameMeta) Reset() { *m = FrameMeta{} } @@ -268,6 +271,34 @@ func (m *ClusterStatus) GetNodes() []*NodeStatus { return nil } +type FrameSchema struct { + Fields []*Field `protobuf:"bytes,1,rep,name=Fields" json:"Fields,omitempty"` +} + +func (m *FrameSchema) Reset() { *m = FrameSchema{} } +func (m *FrameSchema) String() string { return proto.CompactTextString(m) } +func (*FrameSchema) ProtoMessage() {} +func (*FrameSchema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } + +func (m *FrameSchema) GetFields() []*Field { + if m != nil { + return m.Fields + } + return nil +} + +type Field struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` + Min int64 `protobuf:"varint,3,opt,name=Min,proto3" json:"Min,omitempty"` + Max int64 `protobuf:"varint,4,opt,name=Max,proto3" json:"Max,omitempty"` +} + +func (m *Field) Reset() { *m = Field{} } +func (m *Field) String() string { return proto.CompactTextString(m) } +func (*Field) ProtoMessage() {} +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } + func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") @@ -285,6 +316,8 @@ func init() { proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") + proto.RegisterType((*FrameSchema)(nil), "internal.FrameSchema") + proto.RegisterType((*Field)(nil), "internal.Field") } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -364,6 +397,16 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) i += copy(dAtA[i:], m.TimeQuantum) } + if m.RangeEnabled { + dAtA[i] = 0x30 + i++ + if m.RangeEnabled { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -899,6 +942,76 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *FrameSchema) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FrameSchema) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Fields) > 0 { + for _, msg := range m.Fields { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *Field) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Field) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Type) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) + i += copy(dAtA[i:], m.Type) + } + if m.Min != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) + } + if m.Max != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) + } + return i, nil +} + func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) dAtA[offset+1] = uint8(v >> 8) @@ -961,6 +1074,9 @@ func (m *FrameMeta) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.RangeEnabled { + n += 2 + } return n } @@ -1193,6 +1309,38 @@ func (m *ClusterStatus) Size() (n int) { return n } +func (m *FrameSchema) Size() (n int) { + var l int + _ = l + if len(m.Fields) > 0 { + for _, e := range m.Fields { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + +func (m *Field) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Type) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Min != 0 { + n += 1 + sovPrivate(uint64(m.Min)) + } + if m.Max != 0 { + n += 1 + sovPrivate(uint64(m.Max)) + } + return n +} + func sovPrivate(x uint64) (n int) { for { n++ @@ -1469,6 +1617,26 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { } m.TimeQuantum = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RangeEnabled", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.RangeEnabled = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3300,6 +3468,233 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } return nil } +func (m *FrameSchema) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FrameSchema: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FrameSchema: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Fields = append(m.Fields, &Field{}) + if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Field) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Field: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Field: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Type = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Min", wireType) + } + m.Min = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Min |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) + } + m.Max = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Max |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -3408,45 +3803,50 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 640 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x4e, 0x14, 0x4d, - 0x10, 0xfe, 0x67, 0x77, 0x96, 0x7f, 0xa7, 0x08, 0x08, 0x2d, 0x31, 0x23, 0x21, 0x9b, 0x4d, 0x1f, - 0x04, 0x39, 0x70, 0xc0, 0x8b, 0x51, 0x0f, 0x86, 0x5d, 0x0c, 0x93, 0x08, 0xc6, 0x5e, 0xe2, 0xd1, - 0xa4, 0x81, 0x8a, 0x4e, 0x98, 0x9d, 0x59, 0xa7, 0x7b, 0x80, 0xf5, 0xe0, 0x73, 0x98, 0x78, 0xf2, - 0x01, 0x7c, 0x0f, 0x8f, 0x3e, 0x82, 0xc1, 0x17, 0x31, 0x5d, 0xdd, 0x33, 0xb3, 0x0e, 0x22, 0xd1, - 0x5b, 0xd7, 0x57, 0xd5, 0xf5, 0x7d, 0xfd, 0x4d, 0xd5, 0xc0, 0xc2, 0x24, 0x8f, 0xcf, 0xa4, 0xc6, - 0xad, 0x49, 0x9e, 0xe9, 0x8c, 0x75, 0xe3, 0x54, 0x63, 0x9e, 0xca, 0x84, 0xbf, 0x80, 0x20, 0x4a, - 0x4f, 0xf0, 0x62, 0x1f, 0xb5, 0x64, 0x7d, 0x98, 0x1f, 0x64, 0x49, 0x31, 0x4e, 0x9f, 0xcb, 0x23, - 0x4c, 0x42, 0xaf, 0xef, 0x6d, 0x04, 0x62, 0x16, 0x32, 0x15, 0x87, 0xf1, 0x18, 0x5f, 0x16, 0x32, - 0xd5, 0xc5, 0x38, 0x6c, 0xd9, 0x8a, 0x19, 0x88, 0x7f, 0xf1, 0x20, 0x78, 0x96, 0xcb, 0x31, 0x52, - 0xc7, 0x55, 0xe8, 0x8a, 0xec, 0x7c, 0xb6, 0x5d, 0x15, 0xb3, 0x7b, 0xb0, 0x18, 0xa5, 0x67, 0x98, - 0x2b, 0xdc, 0x4d, 0xe5, 0x51, 0x82, 0x27, 0xd4, 0xae, 0x2b, 0x1a, 0x28, 0x5b, 0x83, 0x60, 0x20, - 0x8f, 0xdf, 0xe2, 0xe1, 0x74, 0x82, 0x61, 0x9b, 0x9a, 0xd4, 0x40, 0x95, 0x1d, 0xc5, 0xef, 0x31, - 0xf4, 0xfb, 0xde, 0xc6, 0x82, 0xa8, 0x81, 0xa6, 0xde, 0xce, 0x55, 0xbd, 0x1c, 0x16, 0xa3, 0xf1, - 0x24, 0xcb, 0xb5, 0x40, 0x35, 0xc9, 0x52, 0x85, 0x6c, 0x09, 0xda, 0xbb, 0x79, 0xee, 0xe4, 0x9a, - 0x23, 0xff, 0x00, 0x4b, 0x3b, 0x49, 0x76, 0x7c, 0x3a, 0x94, 0x5a, 0x0a, 0x7c, 0x57, 0xa0, 0xd2, - 0x6c, 0x05, 0x3a, 0x64, 0x9c, 0xab, 0xb3, 0x81, 0x41, 0xe9, 0xf1, 0xce, 0x19, 0x1b, 0x18, 0x94, - 0xee, 0x93, 0x7a, 0x5f, 0xd8, 0xc0, 0xa0, 0xa3, 0x24, 0x3e, 0xb6, 0xaa, 0x7d, 0x61, 0x03, 0xc6, - 0xc0, 0x7f, 0x15, 0xe3, 0xb9, 0x93, 0x4a, 0x67, 0x1e, 0xc1, 0xf2, 0x0c, 0xbf, 0x93, 0x79, 0x07, - 0xe6, 0x44, 0x76, 0x1e, 0x0d, 0x55, 0xe8, 0xf5, 0xdb, 0x1b, 0xbe, 0x70, 0x11, 0x19, 0x42, 0x5f, - 0xcc, 0xa4, 0x5a, 0x94, 0xaa, 0x01, 0x7e, 0x17, 0x3a, 0xe4, 0x8e, 0x79, 0x65, 0x7d, 0xd7, 0x1c, - 0xf9, 0x27, 0x0f, 0x96, 0xf7, 0xe5, 0x05, 0xc9, 0x50, 0x15, 0xcd, 0x1e, 0x04, 0x15, 0x48, 0xd5, - 0xf3, 0xdb, 0x9b, 0x5b, 0xe5, 0xf8, 0x6c, 0x5d, 0xa9, 0xaf, 0x91, 0xdd, 0x54, 0xe7, 0x53, 0x51, - 0x5f, 0x5e, 0x7d, 0x02, 0x8b, 0xbf, 0x26, 0x8d, 0x86, 0x53, 0x9c, 0x96, 0x4e, 0x9f, 0xe2, 0xd4, - 0x78, 0x72, 0x26, 0x93, 0xc2, 0xfa, 0xe7, 0x0b, 0x1b, 0x3c, 0x6a, 0x3d, 0xf4, 0xf8, 0x6b, 0x60, - 0x83, 0x1c, 0xa5, 0x46, 0x6a, 0xb0, 0x8f, 0x4a, 0xc9, 0x37, 0x78, 0xfd, 0x57, 0xb0, 0xce, 0xb6, - 0x66, 0x9d, 0x5d, 0x83, 0x20, 0x52, 0x6e, 0xb6, 0xe8, 0x4b, 0x74, 0x45, 0x0d, 0xf0, 0x4d, 0x60, - 0x43, 0x4c, 0x50, 0xa3, 0x5b, 0x87, 0x3f, 0xf4, 0xe7, 0xa3, 0x52, 0xcb, 0xcd, 0xb5, 0x6c, 0x1d, - 0x7c, 0xb3, 0x09, 0x24, 0x65, 0x7e, 0xfb, 0x76, 0x6d, 0x5d, 0xb5, 0x76, 0x82, 0x0a, 0x78, 0x5c, - 0x36, 0x75, 0xdb, 0x73, 0xc3, 0x03, 0x7f, 0x33, 0x66, 0x25, 0x55, 0xbb, 0x49, 0x55, 0xed, 0xa3, - 0xa3, 0x7a, 0x5a, 0xbe, 0xf5, 0x5f, 0xa9, 0xf8, 0xd0, 0xa1, 0x66, 0x5c, 0x0f, 0x4c, 0xd6, 0xde, - 0xa1, 0xf3, 0xf5, 0x4f, 0x6e, 0xea, 0xf8, 0xec, 0x39, 0xca, 0xbf, 0x6b, 0xd3, 0x70, 0xce, 0xfc, - 0x64, 0xca, 0xc1, 0x72, 0x1b, 0x56, 0xc5, 0x6c, 0x1d, 0xe6, 0x88, 0x55, 0x85, 0x3e, 0xcd, 0xee, - 0xad, 0x86, 0x1a, 0xe1, 0xd2, 0x66, 0x9d, 0xdc, 0x90, 0x77, 0xec, 0x3a, 0xd9, 0x88, 0x4b, 0x80, - 0x83, 0xec, 0x04, 0x47, 0x5a, 0xea, 0x42, 0x19, 0x9d, 0x7b, 0x99, 0xd2, 0xa5, 0x4e, 0x73, 0xa6, - 0x69, 0xd3, 0x52, 0x57, 0x0e, 0x51, 0xc0, 0xee, 0xc3, 0xff, 0xa4, 0x13, 0x55, 0xd8, 0x6e, 0x32, - 0x53, 0x42, 0x94, 0x79, 0xfe, 0x18, 0x16, 0x06, 0x49, 0xa1, 0x34, 0xe6, 0x8e, 0x65, 0x13, 0x3a, - 0x86, 0xb3, 0xdc, 0xb7, 0x95, 0xfa, 0x66, 0x2d, 0x45, 0xd8, 0x92, 0x9d, 0xa5, 0xaf, 0x97, 0x3d, - 0xef, 0xdb, 0x65, 0xcf, 0xfb, 0x7e, 0xd9, 0xf3, 0x3e, 0xfe, 0xe8, 0xfd, 0x77, 0x34, 0x47, 0xff, - 0xf8, 0x07, 0x3f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x5a, 0x1c, 0x79, 0x08, 0xf4, 0x05, 0x00, 0x00, + // 709 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xc1, 0x6e, 0x13, 0x49, + 0x10, 0xdd, 0xb1, 0xc7, 0x5e, 0xbb, 0xbc, 0xc9, 0x26, 0xbd, 0xd1, 0xca, 0x44, 0x91, 0x65, 0xf5, + 0x81, 0x84, 0x1c, 0x72, 0x08, 0x12, 0x42, 0xc0, 0x01, 0xc5, 0x0e, 0x8a, 0x25, 0x1c, 0x44, 0x3b, + 0xe2, 0x88, 0xd4, 0xb1, 0x4b, 0xc9, 0x28, 0xe3, 0x19, 0x33, 0xdd, 0x4e, 0x6c, 0x0e, 0x7c, 0x07, + 0x12, 0x27, 0xfe, 0x86, 0x23, 0xfc, 0x01, 0x0a, 0x3f, 0x82, 0xba, 0xba, 0x67, 0xc6, 0x99, 0x24, + 0x44, 0x70, 0xab, 0x7a, 0x55, 0x5d, 0xf5, 0xea, 0xb9, 0x6a, 0x0c, 0x4b, 0x93, 0x24, 0x38, 0x97, + 0x1a, 0x77, 0x26, 0x49, 0xac, 0x63, 0x56, 0x0b, 0x22, 0x8d, 0x49, 0x24, 0x43, 0xfe, 0x0a, 0xea, + 0xbd, 0x68, 0x84, 0xb3, 0x3e, 0x6a, 0xc9, 0xda, 0xd0, 0xe8, 0xc4, 0xe1, 0x74, 0x1c, 0xbd, 0x94, + 0xc7, 0x18, 0x36, 0xbd, 0xb6, 0xb7, 0x55, 0x17, 0x8b, 0x90, 0xc9, 0x38, 0x0a, 0xc6, 0xf8, 0x7a, + 0x2a, 0x23, 0x3d, 0x1d, 0x37, 0x4b, 0x36, 0x63, 0x01, 0xe2, 0xdf, 0x3c, 0xa8, 0xbf, 0x48, 0xe4, + 0x18, 0xa9, 0xe2, 0x3a, 0xd4, 0x44, 0x7c, 0xb1, 0x58, 0x2e, 0xf3, 0xd9, 0x7d, 0x58, 0xee, 0x45, + 0xe7, 0x98, 0x28, 0xdc, 0x8f, 0xe4, 0x71, 0x88, 0x23, 0x2a, 0x57, 0x13, 0x05, 0x94, 0x6d, 0x40, + 0xbd, 0x23, 0x87, 0xa7, 0x78, 0x34, 0x9f, 0x60, 0xb3, 0x4c, 0x45, 0x72, 0x20, 0x8b, 0x0e, 0x82, + 0xf7, 0xd8, 0xf4, 0xdb, 0xde, 0xd6, 0x92, 0xc8, 0x81, 0x22, 0xdf, 0xca, 0x35, 0xbe, 0x8c, 0xc3, + 0x3f, 0x42, 0x46, 0x27, 0x19, 0x87, 0x2a, 0x71, 0xb8, 0x82, 0x71, 0x0e, 0xcb, 0xbd, 0xf1, 0x24, + 0x4e, 0xb4, 0x40, 0x35, 0x89, 0x23, 0x85, 0x6c, 0x05, 0xca, 0xfb, 0x49, 0xe2, 0x46, 0x32, 0x26, + 0xff, 0x00, 0x2b, 0x7b, 0x61, 0x3c, 0x3c, 0xeb, 0x4a, 0x2d, 0x05, 0xbe, 0x9b, 0xa2, 0xd2, 0x6c, + 0x0d, 0x2a, 0x24, 0xae, 0xcb, 0xb3, 0x8e, 0x41, 0x49, 0x20, 0xa7, 0x9e, 0x75, 0x0c, 0x4a, 0xef, + 0x69, 0x42, 0x5f, 0x58, 0xc7, 0xa0, 0x83, 0x30, 0x18, 0xda, 0xc9, 0x7c, 0x61, 0x1d, 0xc6, 0xc0, + 0x7f, 0x13, 0xe0, 0x85, 0x1b, 0x87, 0x6c, 0xde, 0x83, 0xd5, 0x85, 0xfe, 0x8e, 0xe6, 0xff, 0x50, + 0x15, 0xf1, 0x45, 0xaf, 0xab, 0x9a, 0x5e, 0xbb, 0xbc, 0xe5, 0x0b, 0xe7, 0x91, 0x68, 0xf4, 0xab, + 0x9a, 0x50, 0x89, 0x42, 0x39, 0xc0, 0xef, 0x41, 0x85, 0x14, 0x34, 0x53, 0xe6, 0x6f, 0x8d, 0xc9, + 0x3f, 0x79, 0xb0, 0xda, 0x97, 0x33, 0xa2, 0xa1, 0xb2, 0x36, 0x07, 0x50, 0xcf, 0x40, 0xca, 0x6e, + 0xec, 0x6e, 0xef, 0xa4, 0x2b, 0xb6, 0x73, 0x2d, 0x3f, 0x47, 0xf6, 0x23, 0x9d, 0xcc, 0x45, 0xfe, + 0x78, 0xfd, 0x19, 0x2c, 0x5f, 0x0d, 0x1a, 0x0e, 0x67, 0x38, 0x4f, 0x95, 0x3e, 0xc3, 0xb9, 0xd1, + 0xe4, 0x5c, 0x86, 0x53, 0xab, 0x9f, 0x2f, 0xac, 0xf3, 0xa4, 0xf4, 0xd8, 0xe3, 0x6f, 0x81, 0x75, + 0x12, 0x94, 0x1a, 0xa9, 0x40, 0x1f, 0x95, 0x92, 0x27, 0x78, 0xfb, 0xaf, 0x60, 0x95, 0x2d, 0x2d, + 0x2a, 0xbb, 0x01, 0xf5, 0x9e, 0x72, 0xfb, 0x47, 0xbf, 0x44, 0x4d, 0xe4, 0x00, 0xdf, 0x06, 0xd6, + 0xc5, 0x10, 0x35, 0xba, 0x93, 0xf9, 0x45, 0x7d, 0x3e, 0x48, 0xb9, 0xdc, 0x9d, 0xcb, 0x36, 0xc1, + 0x37, 0xd7, 0x42, 0x54, 0x1a, 0xbb, 0xff, 0xe5, 0xd2, 0x65, 0xa7, 0x29, 0x28, 0x81, 0x07, 0x69, + 0x51, 0x77, 0x61, 0x77, 0x0c, 0x78, 0xc3, 0x9a, 0xa5, 0xad, 0xca, 0xc5, 0x56, 0xd9, 0xcd, 0xba, + 0x56, 0xcf, 0xd3, 0x59, 0xff, 0xb4, 0x15, 0xef, 0x3a, 0xd4, 0xac, 0xeb, 0xa1, 0x89, 0xda, 0x37, + 0x64, 0xdf, 0x3e, 0x72, 0x91, 0xc7, 0x67, 0xcf, 0xb5, 0xfc, 0xbd, 0x32, 0x05, 0xe5, 0xcc, 0x87, + 0x28, 0x5d, 0x2c, 0x77, 0x61, 0x99, 0xcf, 0x36, 0xa1, 0x4a, 0x5d, 0x55, 0xd3, 0xa7, 0xdd, 0xfd, + 0xb7, 0xc0, 0x46, 0xb8, 0xb0, 0x39, 0x27, 0xb7, 0xe4, 0x15, 0x7b, 0x4e, 0xd6, 0xe3, 0x12, 0xe0, + 0x30, 0x1e, 0xe1, 0x40, 0x4b, 0x3d, 0x55, 0x86, 0xe7, 0x41, 0xac, 0x74, 0xca, 0xd3, 0xd8, 0xb4, + 0x6d, 0x5a, 0xea, 0x4c, 0x21, 0x72, 0xd8, 0x03, 0xf8, 0x9b, 0x78, 0xa2, 0x6a, 0x96, 0x8b, 0x9d, + 0x29, 0x20, 0xd2, 0x38, 0x7f, 0x0a, 0x4b, 0x9d, 0x70, 0xaa, 0x34, 0x26, 0xae, 0xcb, 0x36, 0x54, + 0x4c, 0xcf, 0xf4, 0xde, 0xd6, 0xf2, 0x97, 0x39, 0x15, 0x61, 0x53, 0xf8, 0x23, 0x68, 0xd0, 0x04, + 0x83, 0xe1, 0x29, 0x8e, 0x25, 0xcd, 0x1b, 0x60, 0x38, 0x4a, 0xdf, 0x2e, 0xce, 0x6b, 0x70, 0xe1, + 0xc2, 0x7c, 0x00, 0x15, 0xb2, 0x6e, 0x94, 0x9e, 0x81, 0x4f, 0x5f, 0x64, 0x3b, 0x11, 0xd9, 0xe6, + 0x58, 0xfb, 0x41, 0x44, 0x02, 0x97, 0x85, 0x31, 0x09, 0x91, 0x33, 0xfa, 0x7c, 0x19, 0x44, 0xce, + 0xf6, 0x56, 0xbe, 0x5c, 0xb6, 0xbc, 0xaf, 0x97, 0x2d, 0xef, 0xfb, 0x65, 0xcb, 0xfb, 0xf8, 0xa3, + 0xf5, 0xd7, 0x71, 0x95, 0xfe, 0x94, 0x1e, 0xfe, 0x0c, 0x00, 0x00, 0xff, 0xff, 0x5d, 0x3a, 0x0c, + 0x81, 0xa5, 0x06, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index c7c792f4a..be2967880 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -13,6 +13,7 @@ message FrameMeta { string CacheType = 3; uint32 CacheSize = 4; string TimeQuantum = 5; + bool RangeEnabled = 6; } message ImportResponse { @@ -88,3 +89,14 @@ message NodeStatus { message ClusterStatus { repeated NodeStatus Nodes = 1; } + +message FrameSchema { + repeated Field Fields = 1; +} + +message Field { + string Name = 1; + string Type = 2; + int64 Min = 3; + int64 Max = 4; +} diff --git a/internal/public.pb.go b/internal/public.pb.go index 9eb005c17..75d79133f 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -2576,7 +2576,7 @@ func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ // 576 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40, 0x10, 0xa5, 0x63, 0xe7, 0x57, 0xf9, 0x28, 0x6a, 0xf1, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, 0x57, 0x19, 0x69, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xa6, 0x33, 0xb0, 0xf7, 0xcc, 0xb4, 0x06, 0x4b, 0xfe, 0xd1, 0xdd, 0x16, 0xe4, 0x00, 0xec, 0x91, 0xd8, 0x70, 0x03, 0x38, 0x0a, 0x4b, diff --git a/pilosa.go b/pilosa.go index 13ade19ea..978ebaf30 100644 --- a/pilosa.go +++ b/pilosa.go @@ -36,6 +36,13 @@ var ( ErrFrameInverseDisabled = errors.New("frame inverse disabled") ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") + ErrFieldNameRequired = errors.New("field name required") + ErrInvalidFieldType = errors.New("invalid field type") + ErrInvalidFieldRange = errors.New("invalid field range") + ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") + ErrRangeCacheNotAllowed = errors.New("range cache not allowed") + ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") + ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") From 9fd1de8f2966f3ffad42416f5b35ac2dfda1439a Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 21 Jun 2017 14:38:04 -0500 Subject: [PATCH 15/20] handle nil rowID --- handler_test.go | 4 ++-- index.go | 4 +++- index_test.go | 8 ++++---- input_definition.go | 34 ++++++++++++++++++++-------------- input_definition_test.go | 21 ++++++++++++--------- 5 files changed, 41 insertions(+), 30 deletions(-) diff --git a/handler_test.go b/handler_test.go index 45ca9f021..1b8c914ad 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1121,7 +1121,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -1150,7 +1150,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) diff --git a/index.go b/index.go index c0877f0d7..c70c9eec4 100644 --- a/index.go +++ b/index.go @@ -650,7 +650,9 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin return nil, err } - inputDef.LoadDefinition(pb) + if err = inputDef.LoadDefinition(pb); err != nil { + return nil, err + } if err = inputDef.saveMeta(); err != nil { return nil, err } diff --git a/index_test.go b/index_test.go index f948310ca..f4cb534af 100644 --- a/index_test.go +++ b/index_test.go @@ -247,7 +247,7 @@ func TestIndex_CreateInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) @@ -266,7 +266,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -297,7 +297,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -321,7 +321,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} input, err := index.CreateInputDefinition(&def) diff --git a/input_definition.go b/input_definition.go index f05ccb64a..10cfafd68 100644 --- a/input_definition.go +++ b/input_definition.go @@ -25,7 +25,7 @@ import ( "github.com/pilosa/pilosa/internal" ) -var ValidValueDestination = []string{"map", "valueToRow", "stringToBool"} +var ValidValueDestination = []string{"mapping", "value-to-row", "single-row-boolean"} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -101,10 +101,10 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { if err := i.ValidateAction(action); err != nil { return err } - if action.RowID != 0 && action.Frame != ""{ + if action.RowID != 0 && action.Frame != "" { val, ok := countRowID[action.Frame] if ok && val == action.RowID { - return fmt.Errorf("duplicate rowID with other field: %s", action.RowID) + return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) } else { countRowID[action.Frame] = action.RowID } @@ -113,7 +113,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { Frame: action.Frame, ValueDestination: action.ValueDestination, ValueMap: action.ValueMap, - RowID: action.RowID, + RowID: &action.RowID, }) } @@ -176,7 +176,7 @@ func (i *InputDefinition) saveMeta() error { Frame: action.Frame, ValueDestination: action.ValueDestination, ValueMap: action.ValueMap, - RowID: action.RowID, + RowID: convert(action.RowID), } actions = append(actions, actionMeta) } @@ -226,7 +226,7 @@ type Action struct { Frame string `json:"frame,omitempty"` ValueDestination string `json:"valueDestination,omitempty"` ValueMap map[string]uint64 `json:"valueMap,omitempty"` - RowID uint64 `json:"rowID,omitempty"` + RowID *uint64 `json:"rowID,omitempty"` } // Encode converts Action into its internal representation. @@ -235,10 +235,19 @@ func (o *Action) Encode() *internal.Action { Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, - RowID: o.RowID, + RowID: convert(o.RowID), } } +func convert(x *uint64) uint64 { + if x != nil { + return *x + } + var v int64 = -1 + var v2 uint64 = uint64(v) + return v2 +} + // InputFrame defines the frame used in the input definition. type InputFrame struct { Name string `json:"name,omitempty"` @@ -282,17 +291,14 @@ func (i *InputDefinition) ValidateAction(action *internal.Action) error { } switch action.ValueDestination { - case "map": + case "mapping": if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } - case "stringToBool": - if action.RowID == 0 { - return errors.New("rowID required for stringToBool") + case "single-row-boolean": + if int64(action.RowID) == -1 { + return errors.New("rowID required for single-row-boolean") } - default: - return nil } - return nil } diff --git a/input_definition_test.go b/input_definition_test.go index 178b30ea7..d8b95d5ba 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -29,7 +29,7 @@ func TestInputDefinition_Open(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) @@ -102,7 +102,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { // Create Input Definition. input := pilosa.InputDefinition{} frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "ValueToRow", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err := input.LoadDefinition(def) @@ -110,22 +110,25 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "stringToBool", ValueMap: map[string]uint64{"Green": 1}} + act := pilosa.Action{Frame: "f", ValueDestination: "single-row-boolean", ValueMap: map[string]uint64{"Green": 1}} + encodeAction := act.Encode() + field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{encodeAction}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) - if !strings.Contains(err.Error(), "rowID required for stringToBool") { - t.Fatalf("Expected rowID required for stringToBool error, actual error: %s", err) + if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { + t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "map", RowID: 100} + action = internal.Action{Frame: "f", ValueDestination: "mapping", RowID: 100} + field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "valueMap required for map") { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 100} - action1 := internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 101} + action = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} + action1 := internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 0} field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -133,7 +136,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 100} + action1 = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) From 56c16e923ac3e0c7cb242fc5dd4e53c44edd726a Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 21 Jun 2017 14:53:01 -0500 Subject: [PATCH 16/20] add frame required in action --- input_definition.go | 3 +++ input_definition_test.go | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/input_definition.go b/input_definition.go index 10cfafd68..742139a4a 100644 --- a/input_definition.go +++ b/input_definition.go @@ -282,6 +282,9 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { } func (i *InputDefinition) ValidateAction(action *internal.Action) error { + if action.Frame == "" { + return ErrFrameRequired + } validValues := make(map[string]bool) for _, val := range ValidValueDestination { validValues[val] = true diff --git a/input_definition_test.go b/input_definition_test.go index d8b95d5ba..9a1c4a5a4 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -143,4 +143,11 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { if !strings.Contains(err.Error(), "duplicate rowID with other field") { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } + + action = internal.Action{ValueDestination: "single-row-boolean", RowID: 100} + def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} + err = input.LoadDefinition(def) + if !strings.Contains(err.Error(), "frame required") { + t.Fatalf("Expected frame required error, actual error: %s", err) + } } From 8a0609f1fd24bf22f648b0ff13625e4ea5f8b6e9 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 11:11:36 -0500 Subject: [PATCH 17/20] handle nil rowID --- handler.go | 6 ++++- input_definition.go | 48 ++++++++++++++++++++++++---------------- input_definition_test.go | 22 +++++++++--------- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/handler.go b/handler.go index cefb60ec1..9a9617994 100644 --- a/handler.go +++ b/handler.go @@ -1518,7 +1518,11 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - def := req.Encode() + def, err := req.Encode() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } def.Name = inputDefName // Create InputDefinition. diff --git a/input_definition.go b/input_definition.go index 742139a4a..a24163ca6 100644 --- a/input_definition.go +++ b/input_definition.go @@ -25,7 +25,13 @@ import ( "github.com/pilosa/pilosa/internal" ) -var ValidValueDestination = []string{"mapping", "value-to-row", "single-row-boolean"} +const ( + Mapping = "mapping" + ValueToRow = "value-to-row" + SingleRowBool = "single-row-boolean" +) + +var ValidValueDestination = []string{Mapping, ValueToRow, SingleRowBool} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -116,7 +122,6 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { RowID: &action.RowID, }) } - if field.PrimaryKey { numPrimaryKey += 1 } @@ -213,12 +218,17 @@ type Field struct { } // Encode converts Field into its internal representation. -func (o *Field) Encode() *internal.InputDefinitionField { +func (o *Field) Encode() (*internal.InputDefinitionField, error) { field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey} + for _, action := range o.Actions { - field.Actions = append(field.Actions, action.Encode()) + actionEncode, err := action.Encode() + if err != nil { + return nil, err + } + field.Actions = append(field.Actions, actionEncode) } - return &field + return &field, nil } // Action descripes the mapping method for the field in the InputDefinition. @@ -230,22 +240,23 @@ type Action struct { } // Encode converts Action into its internal representation. -func (o *Action) Encode() *internal.Action { +func (o *Action) Encode() (*internal.Action, error) { + if o.RowID == nil && o.ValueDestination == "single-row-boolean" { + return nil, errors.New("rowID required for single-row-boolean") + } return &internal.Action{ Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, RowID: convert(o.RowID), - } + }, nil } func convert(x *uint64) uint64 { if x != nil { return *x } - var v int64 = -1 - var v2 uint64 = uint64(v) - return v2 + return 0 } // InputFrame defines the frame used in the input definition. @@ -261,16 +272,20 @@ type InputDefinitionInfo struct { } // Encode converts InputDefinitionInfo into its internal representation. -func (i *InputDefinitionInfo) Encode() *internal.InputDefinition { +func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) { var def internal.InputDefinition for _, f := range i.Frames { def.Frames = append(def.Frames, &internal.Frame{Name: f.Name, Meta: f.Options.Encode()}) } for _, f := range i.Fields { - def.Fields = append(def.Fields, f.Encode()) + fEncode, err := f.Encode() + if err != nil { + return nil, err + } + def.Fields = append(def.Fields, fEncode) } - return &def + return &def, nil } func (i *InputDefinition) AddFrame(frame InputFrame) error { @@ -292,16 +307,11 @@ func (i *InputDefinition) ValidateAction(action *internal.Action) error { if _, ok := validValues[action.ValueDestination]; !ok { return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) } - switch action.ValueDestination { - case "mapping": + case Mapping: if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } - case "single-row-boolean": - if int64(action.RowID) == -1 { - return errors.New("rowID required for single-row-boolean") - } } return nil } diff --git a/input_definition_test.go b/input_definition_test.go index 9a1c4a5a4..160064e90 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -80,7 +80,10 @@ func TestInputDefinition_Encoding(t *testing.T) { t.Fatal(err) } - internalDef := def.Encode() + internalDef, err := def.Encode() + if err != nil { + t.Fatal(err) + } if internalDef.Frames[0].Name != "event-time" { t.Fatalf("unexpected frame: %v", internalDef) @@ -110,16 +113,13 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - act := pilosa.Action{Frame: "f", ValueDestination: "single-row-boolean", ValueMap: map[string]uint64{"Green": 1}} - encodeAction := act.Encode() - field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{encodeAction}} - def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} - err = input.LoadDefinition(def) + act := pilosa.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, ValueMap: map[string]uint64{"Green": 1}} + _, err = act.Encode() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "mapping", RowID: 100} + action = internal.Action{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) @@ -127,8 +127,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} - action1 := internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 0} + action = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + action1 := internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -136,7 +136,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} + action1 = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -144,7 +144,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - action = internal.Action{ValueDestination: "single-row-boolean", RowID: 100} + action = internal.Action{ValueDestination: pilosa.SingleRowBool, RowID: 100} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "frame required") { From 13b32b0b109e66bea051d3d368871375fcf68874 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 14:12:16 -0500 Subject: [PATCH 18/20] check SingleRowBool for duplicate rowID --- input_definition.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input_definition.go b/input_definition.go index a24163ca6..10179ca31 100644 --- a/input_definition.go +++ b/input_definition.go @@ -107,7 +107,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { if err := i.ValidateAction(action); err != nil { return err } - if action.RowID != 0 && action.Frame != "" { + if action.ValueDestination == SingleRowBool && action.Frame != "" { val, ok := countRowID[action.Frame] if ok && val == action.RowID { return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) From aa2cc13a9cb478b0a15b8376ad99627dc91e2766 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 14:55:29 -0500 Subject: [PATCH 19/20] change Action to InputDefinitionAction in proto --- handler_test.go | 8 +- index_test.go | 16 ++-- input_definition.go | 20 ++--- input_definition_test.go | 32 ++++---- internal/private.pb.go | 161 ++++++++++++++++++++------------------- internal/private.proto | 4 +- 6 files changed, 121 insertions(+), 120 deletions(-) diff --git a/handler_test.go b/handler_test.go index 1b8c914ad..35d57e912 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1121,8 +1121,8 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) if err != nil { @@ -1150,8 +1150,8 @@ func TestHandler_GetInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { diff --git a/index_test.go b/index_test.go index f4cb534af..a1c265ee4 100644 --- a/index_test.go +++ b/index_test.go @@ -247,8 +247,8 @@ func TestIndex_CreateInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { @@ -266,8 +266,8 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) if err != nil { @@ -297,8 +297,8 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) if err != nil { @@ -321,8 +321,8 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} input, err := index.CreateInputDefinition(&def) if err != nil { diff --git a/input_definition.go b/input_definition.go index 10179ca31..53f3caaa2 100644 --- a/input_definition.go +++ b/input_definition.go @@ -103,7 +103,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { countRowID := make(map[string]uint64) for _, field := range pb.Fields { var actions []Action - for _, action := range field.Actions { + for _, action := range field.InputDefinitionActions { if err := i.ValidateAction(action); err != nil { return err } @@ -175,9 +175,9 @@ func (i *InputDefinition) saveMeta() error { var fields []*internal.InputDefinitionField for _, field := range i.fields { - var actions []*internal.Action + var actions []*internal.InputDefinitionAction for _, action := range field.Actions { - actionMeta := &internal.Action{ + actionMeta := &internal.InputDefinitionAction{ Frame: action.Frame, ValueDestination: action.ValueDestination, ValueMap: action.ValueMap, @@ -187,9 +187,9 @@ func (i *InputDefinition) saveMeta() error { } fieldMeta := &internal.InputDefinitionField{ - Name: field.Name, - PrimaryKey: field.PrimaryKey, - Actions: actions, + Name: field.Name, + PrimaryKey: field.PrimaryKey, + InputDefinitionActions: actions, } fields = append(fields, fieldMeta) } @@ -226,7 +226,7 @@ func (o *Field) Encode() (*internal.InputDefinitionField, error) { if err != nil { return nil, err } - field.Actions = append(field.Actions, actionEncode) + field.InputDefinitionActions = append(field.InputDefinitionActions, actionEncode) } return &field, nil } @@ -240,11 +240,11 @@ type Action struct { } // Encode converts Action into its internal representation. -func (o *Action) Encode() (*internal.Action, error) { +func (o *Action) Encode() (*internal.InputDefinitionAction, error) { if o.RowID == nil && o.ValueDestination == "single-row-boolean" { return nil, errors.New("rowID required for single-row-boolean") } - return &internal.Action{ + return &internal.InputDefinitionAction{ Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, @@ -296,7 +296,7 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { return nil } -func (i *InputDefinition) ValidateAction(action *internal.Action) error { +func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) error { if action.Frame == "" { return ErrFrameRequired } diff --git a/input_definition_test.go b/input_definition_test.go index 160064e90..aa6acd5f8 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -29,8 +29,8 @@ func TestInputDefinition_Open(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { @@ -91,10 +91,10 @@ func TestInputDefinition_Encoding(t *testing.T) { t.Fatalf("unexpected frame meta data: %v", internalDef) } else if len(internalDef.Fields) != 2 { t.Fatalf("unexpected number of Fields: %d", len(internalDef.Fields)) - } else if len(internalDef.Fields[1].Actions) != 1 { - t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].Actions) - } else if internalDef.Fields[1].Actions[0].ValueDestination != "mapping" { - t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].Actions[0]) + } else if len(internalDef.Fields[1].InputDefinitionActions) != 1 { + t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].InputDefinitionActions) + } else if internalDef.Fields[1].InputDefinitionActions[0].ValueDestination != "mapping" { + t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].InputDefinitionActions[0]) } } @@ -105,8 +105,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { // Create Input Definition. input := pilosa.InputDefinition{} frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} - field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} + field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err := input.LoadDefinition(def) if !strings.Contains(err.Error(), "invalid ValueDestination") { @@ -119,32 +119,32 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} - field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} + field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "valueMap required for map") { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} - action1 := internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} - field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} + field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "duplicate primaryKey with other field") { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} - field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} + action1 = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "duplicate rowID with other field") { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - action = internal.Action{ValueDestination: pilosa.SingleRowBool, RowID: 100} + action = internal.InputDefinitionAction{ValueDestination: pilosa.SingleRowBool, RowID: 100} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "frame required") { diff --git a/internal/private.pb.go b/internal/private.pb.go index a59d44430..d25558aff 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -25,7 +25,7 @@ Index InputDefinition InputDefinitionField - Action + InputDefinitionAction CreateInputDefinitionMessage DeleteInputDefinitionMessage NodeStatus @@ -273,9 +273,9 @@ func (m *InputDefinition) GetFields() []*InputDefinitionField { } type InputDefinitionField struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - PrimaryKey bool `protobuf:"varint,2,opt,name=PrimaryKey,proto3" json:"PrimaryKey,omitempty"` - Actions []*Action `protobuf:"bytes,3,rep,name=Actions" json:"Actions,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + PrimaryKey bool `protobuf:"varint,2,opt,name=PrimaryKey,proto3" json:"PrimaryKey,omitempty"` + InputDefinitionActions []*InputDefinitionAction `protobuf:"bytes,3,rep,name=InputDefinitionActions" json:"InputDefinitionActions,omitempty"` } func (m *InputDefinitionField) Reset() { *m = InputDefinitionField{} } @@ -283,26 +283,26 @@ func (m *InputDefinitionField) String() string { return proto.Compact func (*InputDefinitionField) ProtoMessage() {} func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } -func (m *InputDefinitionField) GetActions() []*Action { +func (m *InputDefinitionField) GetInputDefinitionActions() []*InputDefinitionAction { if m != nil { - return m.Actions + return m.InputDefinitionActions } return nil } -type Action struct { +type InputDefinitionAction struct { Frame string `protobuf:"bytes,1,opt,name=Frame,proto3" json:"Frame,omitempty"` ValueDestination string `protobuf:"bytes,2,opt,name=ValueDestination,proto3" json:"ValueDestination,omitempty"` ValueMap map[string]uint64 `protobuf:"bytes,3,rep,name=ValueMap" json:"ValueMap,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` RowID uint64 `protobuf:"varint,4,opt,name=RowID,proto3" json:"RowID,omitempty"` } -func (m *Action) Reset() { *m = Action{} } -func (m *Action) String() string { return proto.CompactTextString(m) } -func (*Action) ProtoMessage() {} -func (*Action) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (m *InputDefinitionAction) Reset() { *m = InputDefinitionAction{} } +func (m *InputDefinitionAction) String() string { return proto.CompactTextString(m) } +func (*InputDefinitionAction) ProtoMessage() {} +func (*InputDefinitionAction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } -func (m *Action) GetValueMap() map[string]uint64 { +func (m *InputDefinitionAction) GetValueMap() map[string]uint64 { if m != nil { return m.ValueMap } @@ -392,7 +392,7 @@ func init() { proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*InputDefinition)(nil), "internal.InputDefinition") proto.RegisterType((*InputDefinitionField)(nil), "internal.InputDefinitionField") - proto.RegisterType((*Action)(nil), "internal.Action") + proto.RegisterType((*InputDefinitionAction)(nil), "internal.InputDefinitionAction") proto.RegisterType((*CreateInputDefinitionMessage)(nil), "internal.CreateInputDefinitionMessage") proto.RegisterType((*DeleteInputDefinitionMessage)(nil), "internal.DeleteInputDefinitionMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") @@ -1030,8 +1030,8 @@ func (m *InputDefinitionField) MarshalTo(dAtA []byte) (int, error) { } i++ } - if len(m.Actions) > 0 { - for _, msg := range m.Actions { + if len(m.InputDefinitionActions) > 0 { + for _, msg := range m.InputDefinitionActions { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) @@ -1045,7 +1045,7 @@ func (m *InputDefinitionField) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Action) Marshal() (dAtA []byte, err error) { +func (m *InputDefinitionAction) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1055,7 +1055,7 @@ func (m *Action) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Action) MarshalTo(dAtA []byte) (int, error) { +func (m *InputDefinitionAction) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1538,8 +1538,8 @@ func (m *InputDefinitionField) Size() (n int) { if m.PrimaryKey { n += 2 } - if len(m.Actions) > 0 { - for _, e := range m.Actions { + if len(m.InputDefinitionActions) > 0 { + for _, e := range m.InputDefinitionActions { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } @@ -1547,7 +1547,7 @@ func (m *InputDefinitionField) Size() (n int) { return n } -func (m *Action) Size() (n int) { +func (m *InputDefinitionAction) Size() (n int) { var l int _ = l l = len(m.Frame) @@ -3775,7 +3775,7 @@ func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { m.PrimaryKey = bool(v != 0) case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Actions", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field InputDefinitionActions", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -3799,8 +3799,8 @@ func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Actions = append(m.Actions, &Action{}) - if err := m.Actions[len(m.Actions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.InputDefinitionActions = append(m.InputDefinitionActions, &InputDefinitionAction{}) + if err := m.InputDefinitionActions[len(m.InputDefinitionActions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -3825,7 +3825,7 @@ func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { } return nil } -func (m *Action) Unmarshal(dAtA []byte) error { +func (m *InputDefinitionAction) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3848,10 +3848,10 @@ func (m *Action) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Action: wiretype end group for non-group") + return fmt.Errorf("proto: InputDefinitionAction: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Action: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: InputDefinitionAction: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -4635,58 +4635,59 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 842 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcd, 0x8e, 0x1b, 0x45, - 0x10, 0x66, 0xec, 0xb1, 0x63, 0xd7, 0x6a, 0x77, 0x9d, 0x66, 0x85, 0x26, 0xab, 0x95, 0xb5, 0xea, - 0x03, 0x59, 0x7c, 0xf0, 0x21, 0x48, 0x08, 0x12, 0x0e, 0x10, 0xdb, 0xd1, 0x5a, 0xe0, 0x00, 0xed, - 0x28, 0x47, 0xa4, 0x5e, 0xbb, 0x80, 0xd1, 0x8e, 0x67, 0xcc, 0x74, 0x8f, 0x37, 0xe6, 0xc0, 0x05, - 0x89, 0x67, 0x40, 0xe2, 0x19, 0x78, 0x0f, 0x8e, 0x5c, 0xb9, 0xa1, 0xe5, 0xc2, 0x63, 0xa0, 0xfe, - 0x9b, 0x19, 0x8f, 0xd7, 0x8e, 0x02, 0xb7, 0xae, 0xaf, 0xbe, 0xae, 0xfa, 0xba, 0xa6, 0xaa, 0x6c, - 0x38, 0x5c, 0xa6, 0xe1, 0x8a, 0x4b, 0xec, 0x2f, 0xd3, 0x44, 0x26, 0xa4, 0x15, 0xc6, 0x12, 0xd3, - 0x98, 0x47, 0xf4, 0x0b, 0x68, 0x8f, 0xe3, 0x39, 0xbe, 0x9a, 0xa0, 0xe4, 0xe4, 0x1c, 0x0e, 0x06, - 0x49, 0x94, 0x2d, 0xe2, 0xcf, 0xf9, 0x15, 0x46, 0x81, 0x77, 0xee, 0x5d, 0xb4, 0x59, 0x19, 0x52, - 0x8c, 0x17, 0xe1, 0x02, 0xbf, 0xca, 0x78, 0x2c, 0xb3, 0x45, 0x50, 0x33, 0x8c, 0x12, 0x44, 0x7f, - 0xf3, 0xa0, 0xfd, 0x2c, 0xe5, 0x0b, 0xd4, 0x11, 0x4f, 0xa1, 0xc5, 0x92, 0x9b, 0x72, 0xb8, 0xdc, - 0x26, 0xef, 0xc2, 0xd1, 0x38, 0x5e, 0x61, 0x2a, 0x70, 0x14, 0xf3, 0xab, 0x08, 0xe7, 0x3a, 0x5c, - 0x8b, 0x55, 0x50, 0x72, 0x06, 0xed, 0x01, 0x9f, 0x7d, 0x87, 0x2f, 0xd6, 0x4b, 0x0c, 0xea, 0x3a, - 0x48, 0x01, 0xe4, 0xde, 0x69, 0xf8, 0x03, 0x06, 0xfe, 0xb9, 0x77, 0x71, 0xc8, 0x0a, 0xa0, 0xaa, - 0xb7, 0xb1, 0xad, 0x97, 0xc2, 0xd1, 0x78, 0xb1, 0x4c, 0x52, 0xc9, 0x50, 0x2c, 0x93, 0x58, 0x20, - 0xe9, 0x40, 0x7d, 0x94, 0xa6, 0x56, 0xae, 0x3a, 0xd2, 0x1f, 0xa1, 0xf3, 0x34, 0x4a, 0x66, 0xd7, - 0x43, 0x2e, 0x39, 0xc3, 0xef, 0x33, 0x14, 0x92, 0x9c, 0x40, 0x43, 0x17, 0xce, 0xf2, 0x8c, 0xa1, - 0x50, 0xfd, 0x78, 0x5b, 0x19, 0x63, 0x28, 0x54, 0xdf, 0xd7, 0xea, 0x7d, 0x66, 0x0c, 0x85, 0x4e, - 0xa3, 0x70, 0x66, 0x54, 0xfb, 0xcc, 0x18, 0x84, 0x80, 0xff, 0x32, 0xc4, 0x1b, 0x2b, 0x55, 0x9f, - 0xe9, 0x18, 0xee, 0x97, 0xf2, 0x5b, 0x99, 0xef, 0x40, 0x93, 0x25, 0x37, 0xe3, 0xa1, 0x08, 0xbc, - 0xf3, 0xfa, 0x85, 0xcf, 0xac, 0xa5, 0x0b, 0xa2, 0xbf, 0x98, 0x72, 0xd5, 0xb4, 0xab, 0x00, 0xe8, - 0x03, 0x68, 0xe8, 0xea, 0xa8, 0x57, 0x16, 0x77, 0xd5, 0x91, 0xfe, 0xea, 0xc1, 0xfd, 0x09, 0x7f, - 0xa5, 0x65, 0x88, 0x3c, 0xcd, 0x25, 0xb4, 0x73, 0x50, 0xb3, 0x0f, 0x1e, 0xf5, 0xfa, 0xae, 0x7d, - 0xfa, 0x5b, 0xfc, 0x02, 0x19, 0xc5, 0x32, 0x5d, 0xb3, 0xe2, 0xf2, 0xe9, 0xc7, 0x70, 0xb4, 0xe9, - 0x54, 0x1a, 0xae, 0x71, 0xed, 0x2a, 0x7d, 0x8d, 0x6b, 0x55, 0x93, 0x15, 0x8f, 0x32, 0x53, 0x3f, - 0x9f, 0x19, 0xe3, 0x71, 0xed, 0x43, 0x8f, 0x7e, 0x0d, 0x64, 0x90, 0x22, 0x97, 0xa8, 0x03, 0x4c, - 0x50, 0x08, 0xfe, 0x2d, 0xee, 0xfe, 0x0a, 0xa6, 0xb2, 0xb5, 0x72, 0x65, 0xcf, 0xa0, 0x3d, 0x16, - 0xb6, 0xb7, 0xf4, 0x97, 0x68, 0xb1, 0x02, 0xa0, 0x3d, 0x20, 0x43, 0x8c, 0x50, 0xa2, 0x1d, 0x87, - 0x3d, 0xf1, 0xe9, 0xd4, 0x69, 0x79, 0x3d, 0x97, 0x3c, 0x04, 0x5f, 0x4d, 0x82, 0x96, 0x72, 0xf0, - 0xe8, 0xed, 0xa2, 0x74, 0xf9, 0xd8, 0x31, 0x4d, 0xa0, 0xa1, 0x0b, 0x6a, 0xa7, 0xe7, 0x35, 0x0f, - 0xbc, 0xa3, 0xcd, 0x5c, 0xaa, 0x7a, 0x35, 0x55, 0x3e, 0x8f, 0x36, 0xd5, 0x27, 0xee, 0xad, 0xff, - 0x35, 0x15, 0x1d, 0x5a, 0x54, 0xb5, 0xeb, 0x73, 0xe5, 0x35, 0x77, 0xf4, 0x79, 0xf7, 0x93, 0xab, - 0x3a, 0xfe, 0xf1, 0x6c, 0xca, 0x37, 0x0b, 0x53, 0xa9, 0x9c, 0x5a, 0x32, 0xae, 0xb1, 0xec, 0x84, - 0xe5, 0x36, 0x79, 0x08, 0x4d, 0x9d, 0x55, 0x04, 0xbe, 0xee, 0xdd, 0xe3, 0x8a, 0x1a, 0x66, 0xdd, - 0x6a, 0x9c, 0x6c, 0x93, 0x37, 0xcc, 0x38, 0x19, 0x8b, 0x8c, 0xa0, 0x33, 0x8e, 0x97, 0x99, 0x1c, - 0xe2, 0x37, 0x61, 0x1c, 0xca, 0x30, 0x89, 0x45, 0xd0, 0xd4, 0xa1, 0x1e, 0x94, 0x15, 0x6d, 0x30, - 0xd8, 0xd6, 0x15, 0xfa, 0xb3, 0x07, 0xc7, 0x15, 0x70, 0xc7, 0xa3, 0x9d, 0xde, 0xda, 0x7e, 0xbd, - 0x1f, 0x40, 0xf3, 0x59, 0x88, 0xd1, 0x5c, 0x04, 0x75, 0x4d, 0xec, 0xee, 0x54, 0xa3, 0x69, 0xcc, - 0xb2, 0xe9, 0x0a, 0x4e, 0xee, 0xf2, 0xdf, 0x29, 0xa6, 0x0b, 0xf0, 0x65, 0x1a, 0x2e, 0x78, 0xba, - 0xfe, 0x0c, 0xd7, 0x76, 0x3b, 0x97, 0x10, 0xd2, 0x83, 0x7b, 0x9f, 0xce, 0x4c, 0x49, 0x8c, 0x88, - 0x4e, 0x21, 0xc2, 0x38, 0x98, 0x23, 0xd0, 0x3f, 0x3d, 0x68, 0x9a, 0x73, 0xd1, 0x52, 0x5e, 0xb9, - 0x7b, 0x7b, 0xd0, 0x79, 0xa9, 0xa6, 0x7d, 0x88, 0x42, 0x86, 0x31, 0x57, 0x4c, 0xdb, 0x73, 0x5b, - 0x38, 0x79, 0x0c, 0x2d, 0x8d, 0x4d, 0xf8, 0x72, 0xfb, 0xf9, 0x26, 0x4b, 0xdf, 0x11, 0xcc, 0x1e, - 0xca, 0xf9, 0x2a, 0xbb, 0xde, 0x94, 0x6e, 0xed, 0x6a, 0xe3, 0xf4, 0x09, 0x1c, 0x6e, 0x5c, 0x78, - 0xa3, 0xdd, 0xf4, 0x93, 0x07, 0x67, 0x6e, 0x21, 0x6c, 0x94, 0x76, 0xff, 0x68, 0xb9, 0x92, 0xd7, - 0x4a, 0x25, 0xff, 0x08, 0xa0, 0xb8, 0x6e, 0x27, 0x79, 0x4f, 0xa3, 0x95, 0xc8, 0xf4, 0x12, 0xce, - 0xdc, 0x06, 0xfb, 0x7f, 0x22, 0x28, 0x07, 0x78, 0x9e, 0xcc, 0x71, 0x2a, 0xb9, 0xcc, 0x84, 0x62, - 0x5c, 0x26, 0x42, 0xba, 0xce, 0x50, 0x67, 0xbd, 0x61, 0x25, 0x97, 0xf9, 0x56, 0xd0, 0x06, 0x79, - 0x0f, 0xee, 0xe9, 0xa0, 0xe8, 0xfa, 0xe1, 0xb8, 0x32, 0xb4, 0xcc, 0xf9, 0xe9, 0x13, 0x38, 0x1c, - 0x44, 0x99, 0x90, 0x98, 0xda, 0x2c, 0x3d, 0x68, 0xa8, 0x9c, 0xee, 0x37, 0xe6, 0xa4, 0xb8, 0x59, - 0x48, 0x61, 0x86, 0xf2, 0xb4, 0xf3, 0xfb, 0x6d, 0xd7, 0xfb, 0xe3, 0xb6, 0xeb, 0xfd, 0x75, 0xdb, - 0xf5, 0x7e, 0xf9, 0xbb, 0xfb, 0xd6, 0x55, 0x53, 0xff, 0xaf, 0x79, 0xff, 0xdf, 0x00, 0x00, 0x00, - 0xff, 0xff, 0x41, 0x4c, 0x6a, 0xbc, 0xe8, 0x08, 0x00, 0x00, + // 851 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcd, 0x6e, 0x23, 0x45, + 0x10, 0xa6, 0xed, 0xb1, 0xb1, 0x2b, 0x4a, 0xe2, 0x6d, 0xc2, 0x6a, 0x36, 0x8a, 0x4c, 0xd4, 0x07, + 0x36, 0x44, 0x22, 0x87, 0x45, 0x42, 0xc0, 0x72, 0x80, 0x8d, 0xb3, 0xca, 0x08, 0xb2, 0x40, 0x67, + 0xb5, 0xdc, 0x90, 0x3a, 0x49, 0x01, 0xa3, 0x8c, 0x67, 0xcc, 0x74, 0x4f, 0xb2, 0xe6, 0xc0, 0x05, + 0x89, 0x67, 0x40, 0xe2, 0xc8, 0x99, 0xf7, 0xe0, 0xc8, 0x23, 0xa0, 0x70, 0xe1, 0x0d, 0xb8, 0xa2, + 0xfe, 0x9b, 0x19, 0x8f, 0x7f, 0xa2, 0xc0, 0x6d, 0xea, 0xeb, 0xaf, 0xaa, 0xbe, 0xae, 0xae, 0x2a, + 0x1b, 0xd6, 0x27, 0x79, 0x7c, 0x25, 0x14, 0x1e, 0x4c, 0xf2, 0x4c, 0x65, 0xb4, 0x17, 0xa7, 0x0a, + 0xf3, 0x54, 0x24, 0xec, 0x33, 0xe8, 0x47, 0xe9, 0x05, 0xbe, 0x3c, 0x41, 0x25, 0xe8, 0x2e, 0xac, + 0x1d, 0x66, 0x49, 0x31, 0x4e, 0x3f, 0x15, 0x67, 0x98, 0x84, 0x64, 0x97, 0xec, 0xf5, 0x79, 0x1d, + 0xd2, 0x8c, 0xe7, 0xf1, 0x18, 0xbf, 0x28, 0x44, 0xaa, 0x8a, 0x71, 0xd8, 0xb2, 0x8c, 0x1a, 0xc4, + 0x7e, 0x23, 0xd0, 0x7f, 0x9a, 0x8b, 0x31, 0x9a, 0x88, 0xdb, 0xd0, 0xe3, 0xd9, 0x75, 0x3d, 0x5c, + 0x69, 0xd3, 0x37, 0x61, 0x23, 0x4a, 0xaf, 0x30, 0x97, 0x78, 0x94, 0x8a, 0xb3, 0x04, 0x2f, 0x4c, + 0xb8, 0x1e, 0x6f, 0xa0, 0x74, 0x07, 0xfa, 0x87, 0xe2, 0xfc, 0x5b, 0x7c, 0x3e, 0x9d, 0x60, 0xd8, + 0x36, 0x41, 0x2a, 0xa0, 0x3c, 0x3d, 0x8d, 0xbf, 0xc7, 0x30, 0xd8, 0x25, 0x7b, 0xeb, 0xbc, 0x02, + 0x9a, 0x7a, 0x3b, 0xf3, 0x7a, 0x19, 0x6c, 0x44, 0xe3, 0x49, 0x96, 0x2b, 0x8e, 0x72, 0x92, 0xa5, + 0x12, 0xe9, 0x00, 0xda, 0x47, 0x79, 0xee, 0xe4, 0xea, 0x4f, 0xf6, 0x03, 0x0c, 0x9e, 0x24, 0xd9, + 0xf9, 0xe5, 0x48, 0x28, 0xc1, 0xf1, 0xbb, 0x02, 0xa5, 0xa2, 0x5b, 0xd0, 0x31, 0x85, 0x73, 0x3c, + 0x6b, 0x68, 0xd4, 0x5c, 0xde, 0x55, 0xc6, 0x1a, 0x1a, 0x35, 0xfe, 0x46, 0x7d, 0xc0, 0xad, 0xa1, + 0xd1, 0xd3, 0x24, 0x3e, 0xb7, 0xaa, 0x03, 0x6e, 0x0d, 0x4a, 0x21, 0x78, 0x11, 0xe3, 0xb5, 0x93, + 0x6a, 0xbe, 0x59, 0x04, 0xf7, 0x6a, 0xf9, 0x9d, 0xcc, 0xfb, 0xd0, 0xe5, 0xd9, 0x75, 0x34, 0x92, + 0x21, 0xd9, 0x6d, 0xef, 0x05, 0xdc, 0x59, 0xa6, 0x20, 0xe6, 0xc5, 0xf4, 0x51, 0xcb, 0x1c, 0x55, + 0x00, 0x7b, 0x00, 0x1d, 0x53, 0x1d, 0x7d, 0xcb, 0xca, 0x57, 0x7f, 0xb2, 0x5f, 0x08, 0xdc, 0x3b, + 0x11, 0x2f, 0x8d, 0x0c, 0x59, 0xa6, 0x39, 0x86, 0x7e, 0x09, 0x1a, 0xf6, 0xda, 0xa3, 0xfd, 0x03, + 0xdf, 0x3e, 0x07, 0x73, 0xfc, 0x0a, 0x39, 0x4a, 0x55, 0x3e, 0xe5, 0x95, 0xf3, 0xf6, 0x87, 0xb0, + 0x31, 0x7b, 0xa8, 0x35, 0x5c, 0xe2, 0xd4, 0x57, 0xfa, 0x12, 0xa7, 0xba, 0x26, 0x57, 0x22, 0x29, + 0x6c, 0xfd, 0x02, 0x6e, 0x8d, 0x0f, 0x5a, 0xef, 0x11, 0xf6, 0x15, 0xd0, 0xc3, 0x1c, 0x85, 0x42, + 0x13, 0xe0, 0x04, 0xa5, 0x14, 0xdf, 0xe0, 0xf2, 0x57, 0xb0, 0x95, 0x6d, 0xd5, 0x2b, 0xbb, 0x03, + 0xfd, 0x48, 0xba, 0xde, 0x32, 0x2f, 0xd1, 0xe3, 0x15, 0xc0, 0xf6, 0x81, 0x8e, 0x30, 0x41, 0x85, + 0x6e, 0x1c, 0x56, 0xc4, 0x67, 0xa7, 0x5e, 0xcb, 0xed, 0x5c, 0xfa, 0x10, 0x02, 0x3d, 0x09, 0x46, + 0xca, 0xda, 0xa3, 0xd7, 0xaa, 0xd2, 0x95, 0x63, 0xc7, 0x0d, 0x81, 0xc5, 0x3e, 0xa8, 0x9b, 0x9e, + 0x5b, 0x2e, 0xb8, 0xa0, 0xcd, 0x7c, 0xaa, 0x76, 0x33, 0x55, 0x39, 0x8f, 0x2e, 0xd5, 0x47, 0xfe, + 0xae, 0xff, 0x35, 0x15, 0x1b, 0x39, 0x54, 0xb7, 0xeb, 0x33, 0x7d, 0x6a, 0x7d, 0xcc, 0xf7, 0xf2, + 0x2b, 0x37, 0x75, 0xfc, 0x4d, 0x5c, 0xca, 0xbb, 0x85, 0x69, 0x54, 0x4e, 0x2f, 0x19, 0xdf, 0x58, + 0x6e, 0xc2, 0x4a, 0x9b, 0x3e, 0x84, 0xae, 0xc9, 0x2a, 0xc3, 0xc0, 0xf4, 0xee, 0x66, 0x43, 0x0d, + 0x77, 0xc7, 0x7a, 0x9c, 0x5c, 0x93, 0x77, 0xec, 0x38, 0x59, 0x8b, 0x1e, 0xc1, 0x20, 0x4a, 0x27, + 0x85, 0x1a, 0xe1, 0xd7, 0x71, 0x1a, 0xab, 0x38, 0x4b, 0x65, 0xd8, 0x35, 0xa1, 0x1e, 0xd4, 0x15, + 0xcd, 0x30, 0xf8, 0x9c, 0x0b, 0xfb, 0x89, 0xc0, 0x66, 0x03, 0x5c, 0x72, 0x69, 0xaf, 0xb7, 0xb5, + 0x5a, 0xef, 0xbb, 0xd0, 0x7d, 0x1a, 0x63, 0x72, 0x21, 0xc3, 0xb6, 0x21, 0x0e, 0x97, 0xaa, 0x31, + 0x34, 0xee, 0xd8, 0xec, 0x57, 0x02, 0x5b, 0x8b, 0x08, 0x0b, 0xd5, 0x0c, 0x01, 0x3e, 0xcf, 0xe3, + 0xb1, 0xc8, 0xa7, 0x9f, 0xe0, 0xd4, 0xad, 0xe7, 0x1a, 0x42, 0xbf, 0x84, 0xfb, 0x8d, 0x58, 0x1f, + 0x9f, 0xdb, 0x12, 0x59, 0x51, 0x6f, 0x2c, 0x15, 0x65, 0x79, 0x7c, 0x89, 0x3b, 0xfb, 0x87, 0xc0, + 0xeb, 0x0b, 0x8f, 0xaa, 0x7e, 0x24, 0xf5, 0xd6, 0xdf, 0x87, 0xc1, 0x0b, 0xbd, 0x2a, 0x46, 0x28, + 0x55, 0x9c, 0x0a, 0xcd, 0x74, 0x0d, 0x3b, 0x87, 0xd3, 0x08, 0x7a, 0x06, 0x3b, 0x11, 0x13, 0x27, + 0xf3, 0xed, 0x5b, 0x64, 0x1e, 0x78, 0xbe, 0xdd, 0x69, 0xa5, 0xbb, 0x16, 0x63, 0xb6, 0xae, 0x5f, + 0xe1, 0xc6, 0xd8, 0x7e, 0x0c, 0xeb, 0x33, 0x0e, 0x77, 0xda, 0x73, 0x3f, 0x12, 0xd8, 0xf1, 0xcb, + 0x65, 0x46, 0xca, 0xea, 0x31, 0xf5, 0xaf, 0xd7, 0xaa, 0xbd, 0xde, 0xfb, 0x00, 0x95, 0xbb, 0xdb, + 0x0a, 0x2b, 0x9a, 0xb6, 0x46, 0x66, 0xc7, 0xb0, 0xe3, 0xb7, 0xe1, 0xff, 0x13, 0xc1, 0x04, 0xc0, + 0xb3, 0xec, 0x02, 0x4f, 0x95, 0x50, 0x85, 0xd4, 0x8c, 0xe3, 0x4c, 0x2a, 0xdf, 0x64, 0xfa, 0xdb, + 0x6c, 0x6b, 0x25, 0x54, 0xb9, 0x61, 0x8c, 0x41, 0xdf, 0x82, 0x57, 0x4d, 0x50, 0xf4, 0xbd, 0xb4, + 0xd9, 0x58, 0x00, 0xdc, 0x9f, 0xb3, 0xc7, 0xb0, 0x7e, 0x98, 0x14, 0x52, 0x61, 0xee, 0xb2, 0xec, + 0x43, 0x47, 0xe7, 0xf4, 0xbf, 0x57, 0x5b, 0x95, 0x67, 0x25, 0x85, 0x5b, 0xca, 0x93, 0xc1, 0xef, + 0x37, 0x43, 0xf2, 0xc7, 0xcd, 0x90, 0xfc, 0x79, 0x33, 0x24, 0x3f, 0xff, 0x35, 0x7c, 0xe5, 0xac, + 0x6b, 0xfe, 0x23, 0xbd, 0xf3, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x15, 0xa7, 0xba, 0xef, 0x34, + 0x09, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 3ad12df7c..0d76f3b98 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -89,10 +89,10 @@ message InputDefinition { message InputDefinitionField { string Name = 1; bool PrimaryKey = 2; - repeated Action Actions = 3; + repeated InputDefinitionAction InputDefinitionActions = 3; } -message Action { +message InputDefinitionAction { string Frame = 1; string ValueDestination = 2; map ValueMap = 3; From 9229d772e904928a8fb54d5e101558a6dfd06a56 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 15:18:37 -0500 Subject: [PATCH 20/20] change Field to InputDefinitionField --- input_definition.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/input_definition.go b/input_definition.go index 53f3caaa2..1e8fd4680 100644 --- a/input_definition.go +++ b/input_definition.go @@ -40,7 +40,7 @@ type InputDefinition struct { index string broadcaster Broadcaster frames []InputFrame - fields []Field + fields []InputDefinitionField } // NewInputDefinition returns a new instance of InputDefinition. @@ -61,7 +61,7 @@ func NewInputDefinition(path, index, name string) (*InputDefinition, error) { func (i *InputDefinition) Frames() []InputFrame { return i.frames } // Fields returns fields of the input definition was initialized with. -func (i *InputDefinition) Fields() []Field { return i.fields } +func (i *InputDefinition) Fields() []InputDefinitionField { return i.fields } // Open opens and initializes the InputDefinition from file. func (i *InputDefinition) Open() error { @@ -130,7 +130,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { return errors.New("duplicate primaryKey with other field") } - inputField := Field{ + inputField := InputDefinitionField{ Name: field.Name, PrimaryKey: field.PrimaryKey, Actions: actions, @@ -210,15 +210,15 @@ func (i *InputDefinition) saveMeta() error { return nil } -// Field descripes a single field mapping in the InputDefinition. -type Field struct { +// InputDefinitionField descripes a single field mapping in the InputDefinition. +type InputDefinitionField struct { Name string `json:"name,omitempty"` PrimaryKey bool `json:"primaryKey,omitempty"` Actions []Action `json:"actions,omitempty"` } -// Encode converts Field into its internal representation. -func (o *Field) Encode() (*internal.InputDefinitionField, error) { +// Encode converts InputDefinitionField into its internal representation. +func (o *InputDefinitionField) Encode() (*internal.InputDefinitionField, error) { field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey} for _, action := range o.Actions { @@ -268,7 +268,7 @@ type InputFrame struct { // InputDefinitionInfo the json message format to create an InputDefinition. type InputDefinitionInfo struct { Frames []InputFrame `json:"frames"` - Fields []Field `json:"fields"` + Fields []InputDefinitionField `json:"fields"` } // Encode converts InputDefinitionInfo into its internal representation.