From 164b7619aa8353c61c85576d61d19bfa23f499ad Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 30 May 2018 16:40:21 +0300 Subject: [PATCH 01/78] Removes bench command --- cmd/bench.go | 57 --------------------- cmd/bench_test.go | 54 -------------------- ctl/bench.go | 116 ------------------------------------------- ctl/bench_test.go | 98 ------------------------------------ ctl/import_test.go | 10 ++++ docs/installation.md | 5 -- 6 files changed, 10 insertions(+), 330 deletions(-) delete mode 100644 cmd/bench.go delete mode 100644 cmd/bench_test.go delete mode 100644 ctl/bench.go delete mode 100644 ctl/bench_test.go diff --git a/cmd/bench.go b/cmd/bench.go deleted file mode 100644 index d4b2b8580..000000000 --- a/cmd/bench.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "io" - "os" - - "github.com/spf13/cobra" - - "github.com/pilosa/pilosa/ctl" -) - -var Bencher *ctl.BenchCommand - -func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Bencher = ctl.NewBenchCommand(os.Stdin, os.Stdout, os.Stderr) - benchCmd := &cobra.Command{ - Use: "bench", - Short: "Benchmark operations.", - Long: ` -Executes a benchmark for a given operation against the index. -`, - RunE: func(cmd *cobra.Command, args []string) error { - if err := Bencher.Run(context.Background()); err != nil { - return err - } - return nil - }, - } - flags := benchCmd.Flags() - flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.") - flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.") - flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]") - flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.") - ctl.SetTLSConfig(flags, &Bencher.TLS.CertificatePath, &Bencher.TLS.CertificateKeyPath, &Bencher.TLS.SkipVerify) - - return benchCmd -} - -func init() { - subcommandFns["bench"] = NewBenchCommand -} diff --git a/cmd/bench_test.go b/cmd/bench_test.go deleted file mode 100644 index 4b94d9392..000000000 --- a/cmd/bench_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd_test - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/cmd" -) - -func TestBenchHelp(t *testing.T) { - output, err := ExecNewRootCommand(t, "bench", "--help") - if !strings.Contains(output, "Usage:") || - !strings.Contains(output, "Flags:") || - !strings.Contains(output, "pilosa bench") || err != nil { - t.Fatalf("Command 'bench --help' not working, err: '%v', output: '%s'", err, output) - } -} - -func TestBenchConfig(t *testing.T) { - tests := []commandTest{ - { - args: []string{"bench", "--operation", "set-bit"}, - env: map[string]string{"PILOSA_HOST": "localhost:12345"}, - cfgFileContent: ` -index = "myindex" -frame = "f1" -`, - validation: func() error { - v := validator{} - v.Check(cmd.Bencher.Host, "localhost:12345") - v.Check(cmd.Bencher.Index, "myindex") - v.Check(cmd.Bencher.Frame, "f1") - v.Check(cmd.Bencher.Op, "set-bit") - v.Check(cmd.Bencher.N, 0) - return v.Error() - }, - }, - } - executeDry(t, tests) -} diff --git a/ctl/bench.go b/ctl/bench.go deleted file mode 100644 index 4a71169d9..000000000 --- a/ctl/bench.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "context" - "fmt" - "io" - "math/rand" - "time" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/server" - "github.com/pkg/errors" -) - -// BenchCommand represents a command for benchmarking index operations. -type BenchCommand struct { - // Destination host and port. - Host string - - // Name of the index & frame to execute against. - Index string - Frame string - - // Type of operation and number to execute. - Op string - N int - - // Standard input/output - *pilosa.CmdIO - - TLS server.TLSConfig -} - -// NewBenchCommand returns a new instance of BenchCommand. -func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand { - return &BenchCommand{ - CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), - } -} - -// Run executes the bench command. -func (cmd *BenchCommand) Run(ctx context.Context) error { - // Create a client to the server. - client, err := CommandClient(cmd) - if err != nil { - return errors.Wrap(err, "creating client") - } - - switch cmd.Op { - case "set-bit": - return cmd.runSetBit(ctx, client) - case "": - return errors.New("op required") - default: - return fmt.Errorf("unknown bench op: %q", cmd.Op) - } -} - -// runSetBit executes a benchmark of random SetBit() operations. -func (cmd *BenchCommand) runSetBit(ctx context.Context, client pilosa.InternalClient) error { - if cmd.N == 0 { - return errors.New("operation count required") - } else if cmd.Index == "" { - return pilosa.ErrIndexRequired - } else if cmd.Frame == "" { - return pilosa.ErrFrameRequired - } - - const maxRowID = 1000 - const maxColumnID = 100000 - - startTime := time.Now() - - // Execute operation continuously. - for i := 0; i < cmd.N; i++ { - rowID := rand.Intn(maxRowID) - columnID := rand.Intn(maxColumnID) - - queryRequest := &internal.QueryRequest{ - Query: fmt.Sprintf(`SetBit(row=%d, frame="%s", col=%d)`, rowID, cmd.Frame, columnID), - Remote: false, - } - if _, err := client.Query(ctx, cmd.Index, queryRequest); err != nil { - return err - } - } - - // Print results. - elapsed := time.Since(startTime) - fmt.Fprintf(cmd.Stdout, "Executed %d operations in %s (%0.3f op/sec)\n", cmd.N, elapsed, float64(cmd.N)/elapsed.Seconds()) - - return nil -} - -func (cmd *BenchCommand) TLSHost() string { - return cmd.Host -} - -func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig { - return cmd.TLS -} diff --git a/ctl/bench_test.go b/ctl/bench_test.go deleted file mode 100644 index 766ec8a1a..000000000 --- a/ctl/bench_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "bufio" - "bytes" - "context" - "fmt" - "io" - "os" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pkg/errors" -) - -func TestBenchCommand_InvalidOption(t *testing.T) { - buf := bytes.Buffer{} - stdin, stdout, stderr := GetIO(buf) - - cm := NewBenchCommand(stdin, stdout, stderr) - err := cm.Run(context.Background()) - if errors.Cause(err) != pilosa.ErrHostRequired { - t.Fatalf("Expect err: %s, actual err: %s", pilosa.ErrHostRequired, err) - } - - cm.Host = "localhost:10101" - err = cm.Run(context.Background()) - if err.Error() != "op required" { - t.Fatalf("Expect err: %s, actual err: %s", "op required", err) - } - - cm.Op = "test" - err = cm.Run(context.Background()) - if err.Error() != "unknown bench op: \"test\"" { - t.Fatalf("Expect err: %s, actual err: %s", "unknown bench op: test", err) - } - -} - -func TestBenchCommand_Run(t *testing.T) { - rder := []byte{} - stdin := bytes.NewReader(rder) - r, w, _ := os.Pipe() - - cm := NewBenchCommand(stdin, w, w) - cm.Op = "set-bit" - cm.Host = "localhost:10101" - - err := cm.Run(context.Background()) - if err.Error() != "operation count required" { - t.Fatalf("Expect error: %s, actual err: %s", "operation count required", err) - } - - cm.N = 1 - err = cm.Run(context.Background()) - if err != pilosa.ErrIndexRequired { - t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrIndexRequired, err) - } - - cm.Index = "i" - err = cm.Run(context.Background()) - if err != pilosa.ErrFrameRequired { - t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrFrameRequired, err) - } - - cm.Frame = "f" - err = cm.Run(context.Background()) - w.Close() - var buf bytes.Buffer - io.Copy(&buf, r) - fmt.Println(buf.String()) - if err != nil { - fmt.Println(buf.String()) - } -} - -// declare stdin, stdout, stderr -func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { - rder := []byte{} - stdin := bytes.NewReader(rder) - stdout := bufio.NewWriter(&buf) - stderr := bufio.NewWriter(&buf) - return stdin, stdout, stderr -} diff --git a/ctl/import_test.go b/ctl/import_test.go index 08eea8323..a11770bd8 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -15,6 +15,7 @@ package ctl import ( + "bufio" "bytes" "context" "io" @@ -180,3 +181,12 @@ func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request { } return req } + +// declare stdin, stdout, stderr +func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { + rder := []byte{} + stdin := bytes.NewReader(rder) + stdout := bufio.NewWriter(&buf) + stderr := bufio.NewWriter(&buf) + return stdin, stdout, stderr +} diff --git a/docs/installation.md b/docs/installation.md index 039ea370d..cf743721f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -49,7 +49,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -108,7 +107,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -173,7 +171,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -262,7 +259,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -327,7 +323,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. From 908066a650b4d599f9e8a7711793ae6a0a95637f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 30 May 2018 11:08:49 -0500 Subject: [PATCH 02/78] Exclude/remove CI matrix configurations to speed up CI process. --- .travis.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fbe1e0c9b..4d413ec3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,9 +27,16 @@ deploy: go: "1.10" condition: $GOARCH = amd64 matrix: + # Excluding or allowing failures on non-primary matrix configurations due to long running times. + fast_finish: true allow_failures: - go: master - fast_finish: true + - go: 1.9 + exclude: + - go: 1.9 + env: GOARCH=386 + - go: 1.9 + env: GOARCH=amd64 notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" From c1c89b9ef850e82f339f81b90337505700fc2a63 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 1 Jun 2018 09:18:23 -0500 Subject: [PATCH 03/78] fix generate-config command, use single toml lib The generate-config command was printing a fixed string rather than calling NewConfig() which is the canonical source for default config. I also noticed that we were depending on two different toml libraries, and so collapsed that to a single one. We have to use pelletier rather than BurntSushi because the viper library that we use depends on pelletier. --- Gopkg.lock | 16 +++++++++------- ctl/generate_config.go | 33 +++++++++------------------------ ctl/generate_config_test.go | 2 +- server/server_test.go | 4 ++-- 4 files changed, 21 insertions(+), 34 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 8bb1744bd..0b4a8e9ea 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,12 +1,6 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. -[[projects]] - name = "github.com/BurntSushi/toml" - packages = ["."] - revision = "b26d9c308763d68093482582cea63d69be07a0f0" - version = "v0.3.0" - [[projects]] branch = "master" name = "github.com/CAFxX/gcnotifier" @@ -212,14 +206,22 @@ [[projects]] name = "github.com/shirou/gopsutil" packages = [ + "cpu", "host", "internal/common", "mem", + "net", "process" ] revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" +[[projects]] + branch = "master" + name = "github.com/shirou/w32" + packages = ["."] + revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" + [[projects]] branch = "master" name = "github.com/spf13/afero" @@ -302,6 +304,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "8f633d73d966ca439d2fdf3704a41d8ea59be8ed9a2cab0ab73de4b72c5772ba" + inputs-digest = "325d0fb217ec7f1509186ff947e184f6c8e65941f06000eb110180e65816b1a4" solver-name = "gps-cdcl" solver-version = 1 diff --git a/ctl/generate_config.go b/ctl/generate_config.go index a64678475..a9429ec80 100644 --- a/ctl/generate_config.go +++ b/ctl/generate_config.go @@ -18,9 +18,11 @@ import ( "context" "fmt" "io" - "strings" + "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // GenerateConfigCommand represents a command for printing a default config. @@ -37,28 +39,11 @@ func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *Genera // Run prints out the default config. func (cmd *GenerateConfigCommand) Run(ctx context.Context) error { - fmt.Fprintln(cmd.Stdout, strings.TrimSpace(` -data-dir = "~/.pilosa" -bind = "localhost:10101" -max-writes-per-request = 5000 - -[cluster] - replicas = 1 - hosts = [ - "localhost:10101", - ] - -[anti-entropy] - interval = "10m0s" - -[profile] - cpu = "" - cpu-time = "30s" - -[metric] - service = "statsd" - host = "127.0.0.1:8125" - poll-interval = "0m15s" -`)+"\n") + conf := server.NewConfig() + ret, err := toml.Marshal(*conf) + if err != nil { + return errors.Wrap(err, "unmarshaling default config") + } + fmt.Fprintf(cmd.Stdout, "%s\n", ret) return nil } diff --git a/ctl/generate_config_test.go b/ctl/generate_config_test.go index 56f392bba..26b531e8f 100644 --- a/ctl/generate_config_test.go +++ b/ctl/generate_config_test.go @@ -34,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) { io.Copy(&buf, r) if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), "localhost:10101") { + } else if !strings.Contains(buf.String(), ":10101") { t.Fatalf("Unexpected config: %s", buf.String()) } } diff --git a/server/server_test.go b/server/server_test.go index 9092212b2..edac4b6fd 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -27,7 +27,7 @@ import ( "testing" "testing/quick" - "github.com/BurntSushi/toml" + "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" @@ -374,7 +374,7 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { // ParseConfig parses s into a Config. func ParseConfig(s string) (server.Config, error) { var c server.Config - _, err := toml.Decode(s, &c) + err := toml.Unmarshal([]byte(s), &c) return c, err } From a6a121d19f8e1861f5c5386ebe518cdee4285637 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Sat, 2 Jun 2018 09:59:11 -0500 Subject: [PATCH 04/78] add toml tag to config.Handler --- server/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/config.go b/server/config.go index 663e9e18c..6c7db569a 100644 --- a/server/config.go +++ b/server/config.go @@ -61,7 +61,7 @@ type Config struct { Handler struct { // CORS Allowed Origins AllowedOrigins []string `toml:"allowed-origins"` - } + } `toml:"handler"` // TLS TLS TLSConfig `toml:"tls"` From 4edc80ea62181245b17c24b6991d6e7b24ef8fb2 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 31 May 2018 11:17:34 -0500 Subject: [PATCH 05/78] unexport Field --- api.go | 4 ++-- frame.go | 38 +++++++++++++++++++------------------- handler.go | 4 ++-- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/api.go b/api.go index 252480d52..23d8db316 100644 --- a/api.go +++ b/api.go @@ -489,7 +489,7 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { } // CreateField creates a new BSI field in the given index and frame. -func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { +func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *oField) error { if err := api.validate(apiCreateField); err != nil { return errors.Wrap(err, "validating api method") } @@ -549,7 +549,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str } // Fields returns the fields in the given frame. -func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { +func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*oField, error) { if err := api.validate(apiFields); err != nil { return nil, errors.Wrap(err, "validating api method") } diff --git a/frame.go b/frame.go index ab005edcc..c9151c21b 100644 --- a/frame.go +++ b/frame.go @@ -56,7 +56,7 @@ type Frame struct { cacheType string cacheSize uint32 timeQuantum TimeQuantum - fields []*Field + fields []*oField Logger Logger } @@ -296,7 +296,7 @@ func (f *Frame) Close() error { } // Field returns a field by name. -func (f *Frame) Field(name string) *Field { +func (f *Frame) Field(name string) *oField { f.mu.RLock() defer f.mu.RUnlock() for _, field := range f.fields { @@ -308,7 +308,7 @@ func (f *Frame) Field(name string) *Field { } // Fields returns the fields on the frame. -func (f *Frame) Fields() []*Field { +func (f *Frame) Fields() []*oField { f.mu.RLock() defer f.mu.RUnlock() return f.fields @@ -325,7 +325,7 @@ func (f *Frame) HasField(name string) bool { } // CreateField creates a new field on the frame. -func (f *Frame) CreateField(field *Field) error { +func (f *Frame) CreateField(field *oField) error { f.mu.Lock() defer f.mu.Unlock() @@ -338,7 +338,7 @@ func (f *Frame) CreateField(field *Field) error { } // addField adds a single field to fields. -func (f *Frame) addField(field *Field) error { +func (f *Frame) addField(field *oField) error { if err := ValidateField(field); err != nil { return errors.Wrap(err, "validating field") } else if f.HasField(field.Name) { @@ -357,7 +357,7 @@ func (f *Frame) addField(field *Field) error { } // GetFields returns a list of all the fields in the frame. -func (f *Frame) GetFields() ([]*Field, error) { +func (f *Frame) GetFields() ([]*oField, error) { f.mu.RLock() defer f.mu.RUnlock() @@ -950,7 +950,7 @@ type FrameOptions struct { CacheType string `json:"cacheType,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` - Fields []*Field `json:"fields,omitempty"` + Fields []*oField `json:"fields,omitempty"` } // Encode converts o into its internal representation. @@ -996,8 +996,8 @@ func IsValidFieldType(v string) bool { } } -// Field represents a range field on a frame. -type Field struct { +// oField represents a range field on a frame. +type oField struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` Min int64 `json:"min,omitempty"` @@ -1005,7 +1005,7 @@ type Field struct { } // BitDepth returns the number of bits required to store a value between min & max. -func (f *Field) BitDepth() uint { +func (f *oField) BitDepth() uint { for i := uint(0); i < 63; i++ { if f.Max-f.Min < (1 << i) { return i @@ -1026,7 +1026,7 @@ func (f *Field) BitDepth() uint { // In order to make this work, we effectively need to change the operator to LTE. // Executor.executeFieldRangeSlice() takes this into account and returns // `frag.FieldNotNull(field.BitDepth())` in such instances. -func (f *Field) BaseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { +func (f *oField) BaseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { if op == pql.GT || op == pql.GTE { if value > f.Max { return baseValue, true @@ -1051,7 +1051,7 @@ func (f *Field) BaseValue(op pql.Token, value int64) (baseValue uint64, outOfRan } // BaseValueBetween adjusts the min/max value to align with the range for Field. -func (f *Field) BaseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { +func (f *oField) BaseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { if max < f.Min || min > f.Max { return baseValueMin, baseValueMax, true } @@ -1068,7 +1068,7 @@ func (f *Field) BaseValueBetween(min, max int64) (baseValueMin, baseValueMax uin return baseValueMin, baseValueMax, false } -func ValidateField(f *Field) error { +func ValidateField(f *oField) error { if f.Name == "" { return ErrFieldNameRequired } else if !IsValidFieldType(f.Type) { @@ -1079,7 +1079,7 @@ func ValidateField(f *Field) error { return nil } -func encodeFields(a []*Field) []*internal.Field { +func encodeFields(a []*oField) []*internal.Field { if len(a) == 0 { return nil } @@ -1090,18 +1090,18 @@ func encodeFields(a []*Field) []*internal.Field { return other } -func decodeFields(a []*internal.Field) []*Field { +func decodeFields(a []*internal.Field) []*oField { if len(a) == 0 { return nil } - other := make([]*Field, len(a)) + other := make([]*oField, len(a)) for i := range a { other[i] = decodeField(a[i]) } return other } -func encodeField(f *Field) *internal.Field { +func encodeField(f *oField) *internal.Field { if f == nil { return nil } @@ -1113,11 +1113,11 @@ func encodeField(f *Field) *internal.Field { } } -func decodeField(f *internal.Field) *Field { +func decodeField(f *internal.Field) *oField { if f == nil { return nil } - return &Field{ + return &oField{ Name: f.Name, Type: f.Type, Min: f.Min, diff --git a/handler.go b/handler.go index 9ddabaeeb..5b0e0cb05 100644 --- a/handler.go +++ b/handler.go @@ -621,7 +621,7 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { return } - field := &Field{ + field := &oField{ Name: fieldName, Type: req.Type, Min: req.Min, @@ -696,7 +696,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { } type getFrameFieldsResponse struct { - Fields []*Field `json:"fields,omitempty"` + Fields []*oField `json:"fields,omitempty"` } type deleteFrameFieldResponse struct{} From a60ff2d1e07106f5014b2a83752a2c81f6e8ce81 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 31 May 2018 11:36:40 -0500 Subject: [PATCH 06/78] remove field from handler endpoints --- cluster_test.go | 2 +- handler.go | 96 ------------------------------------------------- 2 files changed, 1 insertion(+), 97 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index 913cd5db3..a2570c41a 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -424,7 +424,7 @@ func TestCluster_ResizeStates(t *testing.T) { // Add Field Data to node0. if err := tc.CreateFrame("i", "fields", FrameOptions{ - Fields: []*Field{ + Fields: []*oField{ { Name: "fld0", Type: FieldTypeInt, diff --git a/handler.go b/handler.go index 5b0e0cb05..d4ed5ab37 100644 --- a/handler.go +++ b/handler.go @@ -169,9 +169,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET") - router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") @@ -608,99 +605,6 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { type deleteFrameResponse struct{} -// handlePostFrameField handles POST /frame/field request. -func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - fieldName := mux.Vars(r)["field"] - - // Decode request. - var req postFrameFieldRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - field := &oField{ - Name: fieldName, - Type: req.Type, - Min: req.Min, - Max: req.Max, - } - - if err := h.API.CreateField(r.Context(), indexName, frameName, field); err != nil { - if errors.Cause(err) == ErrFrameNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type postFrameFieldRequest struct { - Type string `json:"type,omitempty"` - Min int64 `json:"min,omitempty"` - Max int64 `json:"max,omitempty"` -} - -type postFrameFieldResponse struct{} - -// handleDeleteFrameField handles DELETE /frame/field request. -func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - fieldName := mux.Vars(r)["field"] - - if err := h.API.DeleteField(r.Context(), indexName, frameName, fieldName); err != nil { - if errors.Cause(err) == ErrFrameNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - fields, err := h.API.Fields(r.Context(), indexName, frameName) - if err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - fallthrough - case ErrFrameNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: fields}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type getFrameFieldsResponse struct { - Fields []*oField `json:"fields,omitempty"` -} - -type deleteFrameFieldResponse struct{} - // handlePostFrameAttrDiff handles POST /frame/attr/diff requests. func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] From d799967fc60eb22a90febb0036b9f4270ce9471a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 1 Jun 2018 16:16:02 -0500 Subject: [PATCH 07/78] add Type to frame so support BSI frame type "int" --- api.go | 21 --- apimethod_string.go | 8 +- diagnostics.go | 4 +- frame.go | 200 ++++++++++++++------- index.go | 27 +-- internal/private.pb.go | 400 ++++++++++++++++++++++++----------------- internal/private.proto | 5 +- internal/public.pb.go | 35 +++- 8 files changed, 418 insertions(+), 282 deletions(-) diff --git a/api.go b/api.go index 23d8db316..c02aedbf2 100644 --- a/api.go +++ b/api.go @@ -548,25 +548,6 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str return errors.Wrap(err, "sending DeleteField message") } -// Fields returns the fields in the given frame. -func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*oField, error) { - if err := api.validate(apiFields); err != nil { - return nil, errors.Wrap(err, "validating api method") - } - - index := api.Holder.index(indexName) - if index == nil { - return nil, ErrIndexNotFound - } - - frame := index.frame(frameName) - if frame == nil { - return nil, ErrFrameNotFound - } - - return frame.GetFields() -} - // Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { if err := api.validate(apiViews); err != nil { @@ -877,7 +858,6 @@ const ( apiDeleteIndex apiDeleteView apiExportCSV - apiFields apiFragmentBlockData apiFragmentBlocks apiFrameAttrDiff @@ -923,7 +903,6 @@ var methodsNormal = map[apiMethod]struct{}{ apiDeleteIndex: struct{}{}, apiDeleteView: struct{}{}, apiExportCSV: struct{}{}, - apiFields: struct{}{}, apiFragmentBlockData: struct{}{}, apiFragmentBlocks: struct{}{}, apiFrameAttrDiff: struct{}{}, diff --git a/apimethod_string.go b/apimethod_string.go index 2eb2913ae..0fc3822d5 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -2,15 +2,15 @@ package pilosa -import "strconv" +import "fmt" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 87, 101, 114, 126, 135, 155, 172, 188, 197, 211, 219, 235, 253, 261, 281, 294, 308, 323, 340, 353, 373, 381} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 87, 101, 114, 126, 146, 163, 179, 188, 202, 210, 226, 244, 252, 272, 285, 299, 316, 329, 349, 357} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { - return "apiMethod(" + strconv.FormatInt(int64(i), 10) + ")" + return fmt.Sprintf("apiMethod(%d)", i) } return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]] } diff --git a/diagnostics.go b/diagnostics.go index 9219c01ea..1968a4e74 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -227,8 +227,8 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { numIndexes += 1 for _, frame := range index.Frames() { numFrames += 1 - if fields, err := frame.GetFields(); err == nil { - bsiFieldCount += len(fields) + if frame.Type() == FrameTypeInt { + bsiFieldCount += 1 } if frame.TimeQuantum() != "" { timeQuantumEnabled = true diff --git a/frame.go b/frame.go index c9151c21b..e0765096e 100644 --- a/frame.go +++ b/frame.go @@ -31,12 +31,21 @@ import ( // Default frame settings. const ( + DefaultFrameType = FrameTypeSet + DefaultCacheType = CacheTypeRanked // Default ranked frame cache DefaultCacheSize = 50000 ) +// Frame types. +const ( + FrameTypeSet = "set" + FrameTypeInt = "int" + FrameTypeTime = "time" +) + // Frame represents a container for views. type Frame struct { mu sync.RWMutex @@ -53,10 +62,9 @@ type Frame struct { Stats StatsClient // Frame options. - cacheType string - cacheSize uint32 - timeQuantum TimeQuantum - fields []*oField + options FrameOptions + + fields []*oField Logger Logger } @@ -80,10 +88,11 @@ func NewFrame(path, index, name string) (*Frame, error) { broadcaster: NopBroadcaster, Stats: NopStatsClient, - cacheType: DefaultCacheType, - cacheSize: DefaultCacheSize, - //timeQuantum - //fields + options: FrameOptions{ + Type: DefaultFrameType, + CacheType: DefaultCacheType, + CacheSize: DefaultCacheSize, + }, Logger: NopLogger, }, nil @@ -115,9 +124,18 @@ func (f *Frame) MaxSlice() uint64 { return max } +// Type returns the frame type. +func (f *Frame) Type() string { + f.mu.RLock() + defer f.mu.RUnlock() + return f.options.Type +} + // CacheType returns the caching mode for the frame. func (f *Frame) CacheType() string { - return f.cacheType + f.mu.RLock() + defer f.mu.RUnlock() + return f.options.CacheType } // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. @@ -127,12 +145,12 @@ func (f *Frame) SetCacheSize(v uint32) error { defer f.mu.Unlock() // Ignore if no change occurred. - if v == 0 || f.cacheSize == v { + if v == 0 || f.options.CacheSize == v { return nil } // Persist meta data to disk on change. - f.cacheSize = v + f.options.CacheSize = v if err := f.saveMeta(); err != nil { return errors.Wrap(err, "saving") } @@ -142,9 +160,9 @@ func (f *Frame) SetCacheSize(v uint32) error { // CacheSize returns the ranked frame cache size. func (f *Frame) CacheSize() uint32 { - f.mu.Lock() - v := f.cacheSize - f.mu.Unlock() + f.mu.RLock() + v := f.options.CacheSize + f.mu.RUnlock() return v } @@ -152,16 +170,7 @@ func (f *Frame) CacheSize() uint32 { func (f *Frame) Options() FrameOptions { f.mu.RLock() defer f.mu.RUnlock() - return f.options() -} - -func (f *Frame) options() FrameOptions { - return FrameOptions{ - CacheType: f.cacheType, - CacheSize: f.cacheSize, - TimeQuantum: f.timeQuantum, - Fields: f.fields, - } + return f.options } // Open opens and initializes the frame. @@ -176,6 +185,11 @@ func (f *Frame) Open() error { return errors.Wrap(err, "loading meta") } + // Apply the frame options loaded from meta. + if err := f.applyOptions(f.options); err != nil { + return errors.Wrap(err, "applying options") + } + if err := f.openViews(); err != nil { return errors.Wrap(err, "opening views") } @@ -216,7 +230,7 @@ func (f *Frame) openViews() error { name := filepath.Base(fi.Name()) view := f.newView(f.ViewPath(name), name) if err := view.Open(); err != nil { - return fmt.Errorf("open view: view=%s, err=%s", view.Name(), err) + return fmt.Errorf("opening view: view=%s, err=%s", view.Name(), err) } view.RowAttrStore = f.rowAttrStore f.views[view.Name()] = view @@ -232,10 +246,6 @@ func (f *Frame) loadMeta() error { // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) if os.IsNotExist(err) { - f.cacheType = DefaultCacheType - f.cacheSize = DefaultCacheSize - f.timeQuantum = "" - //f.fields return nil } else if err != nil { return errors.Wrap(err, "reading meta") @@ -246,13 +256,12 @@ func (f *Frame) loadMeta() error { } // Copy metadata fields. - f.cacheType = pb.CacheType - if f.cacheType == "" { - f.cacheType = DefaultCacheType - } - f.cacheSize = pb.CacheSize - f.timeQuantum = TimeQuantum(pb.TimeQuantum) - f.fields = decodeFields(pb.Fields) + f.options.Type = pb.Type + f.options.CacheType = pb.CacheType + f.options.CacheSize = pb.CacheSize + f.options.Min = pb.Min + f.options.Max = pb.Max + f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum) return nil } @@ -260,7 +269,7 @@ func (f *Frame) loadMeta() error { // saveMeta writes meta data for the frame. func (f *Frame) saveMeta() error { // Marshal metadata. - fo := f.options() + fo := f.options buf, err := proto.Marshal(fo.Encode()) if err != nil { return errors.Wrap(err, "marshaling") @@ -274,6 +283,60 @@ func (f *Frame) saveMeta() error { return nil } +// applyOptions configures the frame based on opt. +func (f *Frame) applyOptions(opt FrameOptions) error { + switch opt.Type { + case FrameTypeSet, "": + f.options.Type = FrameTypeSet + if opt.CacheType != "" { + f.options.CacheType = opt.CacheType + } + if opt.CacheSize != 0 { + f.options.CacheSize = opt.CacheSize + } + f.options.Min = 0 + f.options.Max = 0 + f.options.TimeQuantum = "" + case FrameTypeInt: + f.options.Type = opt.Type + f.options.CacheType = "" + f.options.CacheSize = 0 + f.options.Min = opt.Min + f.options.Max = opt.Max + f.options.TimeQuantum = "" + + // Create new field. + field := &oField{ + Name: f.name, + Type: FieldTypeInt, + Min: opt.Min, + Max: opt.Max, + } + // Validate field. + if err := ValidateField(field); err != nil { + return err + } + if err := f.CreateField(field); err != nil { + return errors.Wrap(err, "creating field") + } + case FrameTypeTime: + f.options.Type = opt.Type + f.options.CacheType = "" + f.options.CacheSize = 0 + f.options.Min = 0 + f.options.Max = 0 + // Set the time quantum. + if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { + f.Close() + return errors.Wrap(err, "setting time quantum") + } + default: + return errors.New("invalid frame type") + } + + return nil +} + // Close closes the frame and its views. func (f *Frame) Close() error { f.mu.Lock() @@ -307,13 +370,6 @@ func (f *Frame) Field(name string) *oField { return nil } -// Fields returns the fields on the frame. -func (f *Frame) Fields() []*oField { - f.mu.RLock() - defer f.mu.RUnlock() - return f.fields -} - // HasField returns true if a field exists on the frame. func (f *Frame) HasField(name string) bool { for _, fld := range f.fields { @@ -356,19 +412,6 @@ func (f *Frame) addField(field *oField) error { return nil } -// GetFields returns a list of all the fields in the frame. -func (f *Frame) GetFields() ([]*oField, error) { - f.mu.RLock() - defer f.mu.RUnlock() - - err := f.loadMeta() - if err != nil { - return nil, errors.Wrap(err, "loading meta") - } - - return f.fields, nil -} - // DeleteField deletes an existing field on the schema. func (f *Frame) DeleteField(name string) error { f.mu.Lock() @@ -410,7 +453,7 @@ func (f *Frame) deleteField(name string) error { func (f *Frame) TimeQuantum() TimeQuantum { f.mu.Lock() defer f.mu.Unlock() - return f.timeQuantum + return f.options.TimeQuantum } // SetTimeQuantum sets the time quantum for the frame. @@ -424,7 +467,7 @@ func (f *Frame) SetTimeQuantum(q TimeQuantum) error { } // Update value on frame. - f.timeQuantum = q + f.options.TimeQuantum = q // Persist meta data to disk. if err := f.saveMeta(); err != nil { @@ -526,8 +569,8 @@ func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) { } func (f *Frame) newView(path, name string) *View { - view := NewView(path, f.index, f.name, name, f.cacheSize) - view.cacheType = f.cacheType + view := NewView(path, f.index, f.name, name, f.options.CacheSize) + view.cacheType = f.options.CacheType view.Logger = f.Logger view.RowAttrStore = f.rowAttrStore view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) @@ -918,7 +961,7 @@ func encodeFrames(a []*Frame) []*internal.Frame { // encodeFrame converts f into its internal representation. func encodeFrame(f *Frame) *internal.Frame { - fo := f.options() + fo := f.options return &internal.Frame{ Name: f.name, Meta: fo.Encode(), @@ -947,10 +990,31 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { + Type string `json:"type,omitempty"` CacheType string `json:"cacheType,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` - Fields []*oField `json:"fields,omitempty"` + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` // TODO travis: rename this Quantum? +} + +// Validate ensures that FrameOption values are valid. +func (o *FrameOptions) Validate() error { + switch o.Type { + case FrameTypeSet, "": + // TODO: cacheType, cacheSize validation + case FrameTypeInt: + if o.Min > o.Max { + return ErrInvalidFieldRange + } + case FrameTypeTime: + if o.TimeQuantum == "" || !o.TimeQuantum.Valid() { + return ErrInvalidTimeQuantum + } + default: + return errors.New("invalid frame type") + } + return nil } // Encode converts o into its internal representation. @@ -963,10 +1027,12 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { return nil } return &internal.FrameMeta{ + Type: o.Type, CacheType: o.CacheType, CacheSize: o.CacheSize, + Min: o.Min, + Max: o.Max, TimeQuantum: string(o.TimeQuantum), - Fields: encodeFields(o.Fields), } } @@ -975,10 +1041,12 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { return nil } return &FrameOptions{ + Type: options.Type, CacheType: options.CacheType, CacheSize: options.CacheSize, + Min: options.Min, + Max: options.Max, TimeQuantum: TimeQuantum(options.TimeQuantum), - Fields: decodeFields(options.Fields), } } diff --git a/index.go b/index.go index 534930cd5..7829903aa 100644 --- a/index.go +++ b/index.go @@ -299,11 +299,9 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, ErrInvalidCacheType } - // Validate fields. - for _, field := range opt.Fields { - if err := ValidateField(field); err != nil { - return nil, err - } + // Validate options. + if err := opt.Validate(); err != nil { + return nil, errors.Wrap(err, "validating options") } // Initialize frame. @@ -317,25 +315,12 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, errors.Wrap(err, "opening") } - // Set the time quantum. - if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { + // Apply frame options. + if err := f.applyOptions(opt); err != nil { f.Close() - return nil, errors.Wrap(err, "setting time quantum") + return nil, errors.Wrap(err, "applying options") } - // Set cache type. - if opt.CacheType == "" { - opt.CacheType = DefaultCacheType - } - f.cacheType = opt.CacheType - - if opt.CacheSize != 0 { - f.cacheSize = opt.CacheSize - } - - // Set fields. - f.fields = opt.Fields - if err := f.saveMeta(); err != nil { f.Close() return nil, errors.Wrap(err, "saving meta") diff --git a/internal/private.pb.go b/internal/private.pb.go index 35b452dce..20909445c 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,5 +1,6 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. +// Code generated by protoc-gen-gogo. // source: private.proto +// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -70,10 +71,12 @@ func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } type FrameMeta struct { - CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` - CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` - TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"` + Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` + CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` + CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` + Min int64 `protobuf:"varint,9,opt,name=Min,proto3" json:"Min,omitempty"` + Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"` + TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } func (m *FrameMeta) Reset() { *m = FrameMeta{} } @@ -81,6 +84,13 @@ func (m *FrameMeta) String() string { return proto.CompactTextString( func (*FrameMeta) ProtoMessage() {} func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } +func (m *FrameMeta) GetType() string { + if m != nil { + return m.Type + } + return "" +} + func (m *FrameMeta) GetCacheType() string { if m != nil { return m.CacheType @@ -95,6 +105,20 @@ func (m *FrameMeta) GetCacheSize() uint32 { return 0 } +func (m *FrameMeta) GetMin() int64 { + if m != nil { + return m.Min + } + return 0 +} + +func (m *FrameMeta) GetMax() int64 { + if m != nil { + return m.Max + } + return 0 +} + func (m *FrameMeta) GetTimeQuantum() string { if m != nil { return m.TimeQuantum @@ -102,13 +126,6 @@ func (m *FrameMeta) GetTimeQuantum() string { return "" } -func (m *FrameMeta) GetFields() []*Field { - if m != nil { - return m.Fields - } - return nil -} - type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` } @@ -1052,17 +1069,21 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) i += copy(dAtA[i:], m.TimeQuantum) } - if len(m.Fields) > 0 { - for _, msg := range m.Fields { - dAtA[i] = 0x3a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } + if len(m.Type) > 0 { + dAtA[i] = 0x42 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) + i += copy(dAtA[i:], m.Type) + } + if m.Min != 0 { + dAtA[i] = 0x48 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) + } + if m.Max != 0 { + dAtA[i] = 0x50 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) } return i, nil } @@ -2228,6 +2249,24 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Private(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2257,11 +2296,15 @@ func (m *FrameMeta) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if len(m.Fields) > 0 { - for _, e := range m.Fields { - l = e.Size() - n += 1 + l + sovPrivate(uint64(l)) - } + l = len(m.Type) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Min != 0 { + n += 1 + sovPrivate(uint64(m.Min)) + } + if m.Max != 0 { + n += 1 + sovPrivate(uint64(m.Max)) } return n } @@ -2939,11 +2982,11 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { } m.TimeQuantum = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 7: + case 8: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -2953,23 +2996,59 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex > l { return io.ErrUnexpectedEOF } - m.Fields = append(m.Fields, &Field{}) - if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.Type = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Min", wireType) + } + m.Min = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Min |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) + } + m.Max = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Max |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3586,14 +3665,51 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey if m.Standard == nil { m.Standard = make(map[string]uint64) } - var mapkey string - var mapvalue uint64 - for iNdEx < postIndex { - entryPreIndex := iNdEx - var wire uint64 + if iNdEx < postIndex { + var valuekey uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3603,69 +3719,31 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + valuekey |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - fieldNum := int32(wire >> 3) - if fieldNum == 1 { - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } + var mapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { + if iNdEx >= l { return io.ErrUnexpectedEOF } - mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey - } else if fieldNum == 2 { - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break } - } else { - iNdEx = entryPreIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > postIndex { - return io.ErrUnexpectedEOF - } - iNdEx += skippy } + m.Standard[mapkey] = mapvalue + } else { + var mapvalue uint64 + m.Standard[mapkey] = mapvalue } - m.Standard[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -6997,70 +7075,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1035 bytes of a gzipped FileDescriptorProto + // 1029 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1b, 0x45, - 0x18, 0x67, 0xbd, 0x6b, 0x27, 0xfe, 0x8c, 0x53, 0x67, 0x5a, 0xc2, 0x16, 0xa1, 0x60, 0x46, 0x45, - 0x0d, 0x1c, 0xa2, 0x92, 0x5e, 0x78, 0x55, 0x8a, 0x12, 0xa7, 0x62, 0x11, 0x89, 0x60, 0x36, 0xe9, - 0x01, 0x89, 0xc3, 0xd4, 0x1e, 0xa5, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x1e, 0x3d, 0x70, 0x85, - 0x0b, 0x17, 0x4e, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x02, 0x0a, 0xff, 0x08, 0x9a, 0x6f, 0x66, 0x1f, - 0xf1, 0xa3, 0xa9, 0x4c, 0x6f, 0xfb, 0xbd, 0x5f, 0xbf, 0xef, 0x9b, 0x85, 0xee, 0x24, 0x8d, 0xce, - 0xb9, 0x12, 0xdb, 0x93, 0x54, 0x2a, 0x49, 0x56, 0xa3, 0x44, 0x89, 0x34, 0xe1, 0x31, 0xed, 0x40, - 0x3b, 0x48, 0x46, 0xe2, 0xf2, 0x50, 0x28, 0x4e, 0x7f, 0x77, 0xa0, 0xfd, 0x34, 0xe5, 0x63, 0xa1, - 0x29, 0xf2, 0x3e, 0xb4, 0xf7, 0xf9, 0xf0, 0x85, 0x38, 0xbe, 0x9a, 0x08, 0xdf, 0xed, 0x3b, 0x5b, - 0x6d, 0x56, 0x31, 0x4a, 0x69, 0x18, 0xbd, 0x14, 0xbe, 0xd7, 0x77, 0xb6, 0xba, 0xac, 0x62, 0x90, - 0x3e, 0x74, 0x8e, 0xa3, 0xb1, 0xf8, 0x3e, 0xe7, 0x89, 0xca, 0xc7, 0x7e, 0x13, 0xad, 0xeb, 0x2c, - 0xf2, 0x10, 0x5a, 0x4f, 0x23, 0x11, 0x8f, 0x32, 0x7f, 0xa5, 0xef, 0x6e, 0x75, 0x76, 0xee, 0x6c, - 0x17, 0x39, 0x6d, 0x23, 0x9f, 0x59, 0x31, 0xa5, 0xb0, 0x16, 0x8c, 0x27, 0x32, 0x55, 0x4c, 0x64, - 0x13, 0x99, 0x64, 0x82, 0xf4, 0xc0, 0x3d, 0x48, 0x53, 0xdf, 0x41, 0xa7, 0xfa, 0x93, 0xfe, 0x0c, - 0xbd, 0xbd, 0x58, 0x0e, 0xcf, 0x06, 0x5c, 0x71, 0x26, 0x7e, 0xca, 0x45, 0xa6, 0xc8, 0x3d, 0x68, - 0x62, 0x65, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x15, 0xfa, 0x0d, 0xc3, 0x45, 0x42, 0x73, 0xd1, 0x1e, - 0xcb, 0xf4, 0x98, 0x21, 0x34, 0x37, 0x8c, 0xa3, 0xa1, 0x29, 0xcf, 0x63, 0x86, 0x20, 0x04, 0xbc, - 0x67, 0x91, 0xb8, 0xb0, 0x35, 0xe1, 0x37, 0x0d, 0x60, 0xbd, 0x16, 0xdf, 0xa6, 0xb9, 0x01, 0x2d, - 0x26, 0x2f, 0x82, 0x41, 0xe6, 0x3b, 0x7d, 0x77, 0xcb, 0x63, 0x96, 0xc2, 0xce, 0xc9, 0x38, 0x1f, - 0x27, 0x5a, 0xd4, 0x40, 0x51, 0xc5, 0xa0, 0xf7, 0xa1, 0x89, 0x6d, 0xd4, 0x55, 0x56, 0xb6, 0xfa, - 0x93, 0xfe, 0xe2, 0x40, 0xfb, 0x90, 0x5f, 0x62, 0x1a, 0x19, 0x79, 0x02, 0xab, 0xa1, 0xe2, 0xc9, - 0x88, 0xa7, 0x23, 0x54, 0xea, 0xec, 0x7c, 0x58, 0xb5, 0xb0, 0x54, 0xdb, 0x2e, 0x74, 0x0e, 0x12, - 0x95, 0x5e, 0xb1, 0xd2, 0xe4, 0xbd, 0x2f, 0xa1, 0x7b, 0x43, 0xa4, 0xe3, 0x9d, 0x89, 0xab, 0xa2, - 0xab, 0x67, 0xe2, 0x4a, 0xd7, 0x7f, 0xce, 0xe3, 0xdc, 0xf4, 0xca, 0x63, 0x86, 0xf8, 0xa2, 0xf1, - 0x99, 0x43, 0x77, 0x81, 0xec, 0xa7, 0x82, 0x2b, 0x81, 0x41, 0x0e, 0x45, 0x96, 0xf1, 0x53, 0xb1, - 0xb8, 0xe3, 0xa6, 0x8b, 0x8d, 0x5a, 0x17, 0xe9, 0x27, 0x40, 0x06, 0x22, 0x16, 0x4a, 0x58, 0xf4, - 0xbd, 0xc2, 0x03, 0x0d, 0x8b, 0x68, 0xb7, 0xeb, 0x92, 0x87, 0xe0, 0x69, 0xf0, 0x62, 0xb0, 0xce, - 0xce, 0xdd, 0xaa, 0x23, 0x25, 0xca, 0x19, 0x2a, 0xd0, 0xa8, 0x70, 0x6a, 0x01, 0x7f, 0x4b, 0x09, - 0x73, 0x40, 0x53, 0x84, 0x72, 0xa7, 0x43, 0x95, 0x2b, 0x64, 0x43, 0xed, 0x16, 0xb5, 0x2e, 0x1b, - 0x8a, 0x9e, 0x96, 0xc9, 0xea, 0x9d, 0x58, 0x26, 0xd9, 0x8f, 0xa0, 0x89, 0xb6, 0x36, 0xdb, 0x99, - 0x6d, 0x33, 0x52, 0xfa, 0xac, 0x4c, 0x75, 0xd9, 0x40, 0xf7, 0xea, 0x81, 0xda, 0x85, 0xdf, 0x1f, - 0xac, 0xae, 0xde, 0x9e, 0x23, 0x6d, 0x63, 0x3c, 0xe1, 0xf7, 0xe2, 0x99, 0x4d, 0x35, 0x52, 0xfb, - 0xd6, 0xeb, 0x96, 0xf9, 0x6e, 0xdf, 0xd5, 0xbe, 0x91, 0xa0, 0x8f, 0xa1, 0x15, 0x0e, 0x5f, 0x88, - 0x31, 0x27, 0x1f, 0xc3, 0x0a, 0xa6, 0x26, 0x32, 0xbb, 0x11, 0x77, 0xa6, 0xe6, 0xcf, 0x0a, 0x39, - 0x1d, 0xd8, 0x92, 0x16, 0x24, 0xd4, 0xc2, 0xd0, 0x99, 0xef, 0xcd, 0xdc, 0x26, 0xcd, 0x67, 0x56, - 0x4c, 0x0f, 0xc0, 0x3d, 0x61, 0x81, 0xde, 0x74, 0xcc, 0xa0, 0xf0, 0x62, 0x29, 0xed, 0xfb, 0x6b, - 0x99, 0x29, 0xdb, 0x20, 0xfc, 0xd6, 0xbc, 0xef, 0x64, 0xaa, 0xb0, 0x3d, 0x5d, 0x86, 0xdf, 0xf4, - 0x47, 0xf0, 0x8e, 0xe4, 0x48, 0x90, 0x35, 0x68, 0x04, 0x03, 0xeb, 0xa3, 0x11, 0x0c, 0xc8, 0x07, - 0xe8, 0xde, 0xf6, 0xa5, 0x5b, 0x25, 0x71, 0xc2, 0x02, 0x86, 0x81, 0x1f, 0x40, 0x37, 0xc8, 0xf6, - 0xa5, 0x4c, 0x47, 0x51, 0xc2, 0x95, 0x4c, 0xd1, 0xeb, 0x2a, 0xbb, 0xc9, 0xa4, 0xbb, 0xd0, 0xd3, - 0xee, 0x43, 0xc5, 0x55, 0x89, 0xbe, 0x0d, 0x68, 0x69, 0x5e, 0x19, 0xce, 0x52, 0xb8, 0xad, 0x5a, - 0xaf, 0x18, 0x2a, 0x12, 0xf4, 0x5b, 0xe3, 0xe1, 0xe0, 0x5c, 0x24, 0xaa, 0x06, 0x0a, 0xa4, 0xd1, - 0x41, 0x97, 0x19, 0x82, 0x50, 0x53, 0x8a, 0xcd, 0x79, 0xad, 0xca, 0x59, 0x73, 0x19, 0xca, 0xe8, - 0x6f, 0x0e, 0x40, 0x91, 0x50, 0x9e, 0x95, 0x26, 0xce, 0x62, 0x13, 0xf2, 0x69, 0xed, 0xf2, 0xcd, - 0xe2, 0xa4, 0x14, 0xb1, 0xda, 0x7d, 0xdc, 0x2a, 0x60, 0x61, 0x21, 0xdf, 0xab, 0xf4, 0x0d, 0xdf, - 0x8e, 0x49, 0x9f, 0x82, 0xee, 0x7e, 0x9c, 0x67, 0x4a, 0xa4, 0x36, 0x23, 0x7d, 0xa1, 0x0d, 0xa3, - 0xec, 0x4f, 0xc5, 0x98, 0xdf, 0x22, 0xf2, 0x00, 0x9a, 0x3a, 0x53, 0x83, 0xcd, 0xd9, 0x32, 0x8c, - 0x90, 0x86, 0x76, 0x3b, 0xe6, 0xc2, 0x8e, 0x80, 0x87, 0x6f, 0xad, 0x85, 0x0b, 0x3e, 0xb3, 0x3d, - 0x70, 0x0f, 0xa3, 0x04, 0x4b, 0x70, 0x99, 0xfe, 0x44, 0x0e, 0xbf, 0xc4, 0x37, 0x49, 0x73, 0xb8, - 0xbe, 0x8f, 0xeb, 0xe6, 0x3a, 0xe8, 0x7d, 0x58, 0x66, 0x67, 0x8b, 0x27, 0xcd, 0xad, 0x3d, 0x69, - 0x21, 0xac, 0x9b, 0x4b, 0xf0, 0x26, 0x9d, 0xfe, 0xd9, 0x80, 0x75, 0x26, 0xb2, 0xe8, 0xa5, 0x08, - 0x92, 0x4c, 0xa5, 0xf9, 0x50, 0x45, 0x32, 0xd1, 0xf6, 0xdf, 0xc8, 0xe7, 0xb6, 0xd5, 0x2e, 0x33, - 0xc4, 0xeb, 0x20, 0x89, 0x3c, 0x82, 0xce, 0x34, 0xfa, 0x67, 0x55, 0xeb, 0x2a, 0xe4, 0x11, 0xac, - 0x84, 0x32, 0x4f, 0x87, 0xe5, 0x6e, 0x6f, 0x54, 0xda, 0x26, 0x33, 0x23, 0x66, 0x85, 0x5a, 0x0d, - 0x47, 0xcd, 0x57, 0xe3, 0x88, 0x3c, 0x99, 0xc2, 0x91, 0xdf, 0x42, 0x83, 0x77, 0x2b, 0x83, 0x1b, - 0x62, 0x76, 0x53, 0x9b, 0xfe, 0xea, 0xc0, 0xdb, 0xf5, 0x14, 0x5e, 0x6b, 0x31, 0xca, 0x89, 0x34, - 0xe6, 0x4e, 0xc4, 0x9d, 0x37, 0x11, 0xaf, 0x9a, 0x48, 0xf5, 0x3a, 0x37, 0xeb, 0xaf, 0xf3, 0x19, - 0xdc, 0x9f, 0x19, 0xd3, 0xbe, 0x1c, 0x4f, 0x34, 0x1e, 0xfe, 0xc7, 0xb8, 0xf4, 0xc9, 0x48, 0x53, - 0x3b, 0xa8, 0x36, 0x33, 0x04, 0xfd, 0x1c, 0xde, 0x09, 0x85, 0xaa, 0x0d, 0xa9, 0x40, 0x5b, 0x1f, - 0xdc, 0x23, 0x71, 0xb1, 0xa0, 0x7c, 0x2d, 0xa2, 0x5f, 0x81, 0x7f, 0x32, 0x19, 0x71, 0x25, 0x96, - 0xb2, 0xde, 0x83, 0xd5, 0x63, 0x39, 0x91, 0xb1, 0x3c, 0xbd, 0xba, 0x65, 0xe5, 0x7d, 0x58, 0x31, - 0xf7, 0xd1, 0xfc, 0xb0, 0xb5, 0x59, 0x41, 0xd2, 0xbb, 0x1a, 0xd0, 0x43, 0x1e, 0x0f, 0xf3, 0x58, - 0xa7, 0xa1, 0xff, 0xdc, 0xb2, 0xbd, 0xde, 0x5f, 0xd7, 0x9b, 0xce, 0xdf, 0xd7, 0x9b, 0xce, 0x3f, - 0xd7, 0x9b, 0xce, 0x1f, 0xff, 0x6e, 0xbe, 0xf5, 0xbc, 0x85, 0xff, 0xdd, 0x8f, 0xff, 0x0b, 0x00, - 0x00, 0xff, 0xff, 0xd3, 0x15, 0x68, 0xea, 0x88, 0x0b, 0x00, 0x00, + 0x18, 0x67, 0xbd, 0x6b, 0x27, 0xfe, 0x82, 0xd3, 0x64, 0x5a, 0xc2, 0x16, 0xa1, 0x60, 0x46, 0x45, + 0x18, 0x0e, 0x51, 0x69, 0x2f, 0xbc, 0x2a, 0x45, 0xb1, 0x83, 0x58, 0x44, 0x22, 0x98, 0x4d, 0x7a, + 0x40, 0xe2, 0x30, 0xb5, 0x47, 0xe9, 0x2a, 0xeb, 0x1d, 0xb3, 0x3b, 0x9b, 0xc4, 0x3d, 0x70, 0x85, + 0x0b, 0x77, 0xc4, 0x8d, 0xff, 0x86, 0x23, 0x7f, 0x02, 0x0a, 0xff, 0x08, 0x9a, 0x6f, 0x66, 0x1f, + 0xf1, 0xa3, 0xa9, 0x4c, 0x6f, 0xf3, 0xbd, 0x5f, 0xbf, 0x6f, 0x66, 0xa0, 0x33, 0x49, 0xa3, 0x0b, + 0xae, 0xc4, 0xde, 0x24, 0x95, 0x4a, 0x92, 0xf5, 0x28, 0x51, 0x22, 0x4d, 0x78, 0x4c, 0x37, 0xa0, + 0x1d, 0x24, 0x23, 0x71, 0x75, 0x24, 0x14, 0xa7, 0x7f, 0x3a, 0xd0, 0xfe, 0x2a, 0xe5, 0x63, 0xa1, + 0x29, 0xf2, 0x2e, 0xb4, 0xfb, 0x7c, 0xf8, 0x5c, 0x9c, 0x4c, 0x27, 0xc2, 0x77, 0xbb, 0x4e, 0xaf, + 0xcd, 0x2a, 0x46, 0x29, 0x0d, 0xa3, 0x17, 0xc2, 0xf7, 0xba, 0x4e, 0xaf, 0xc3, 0x2a, 0x06, 0xe9, + 0xc2, 0xc6, 0x49, 0x34, 0x16, 0xdf, 0xe7, 0x3c, 0x51, 0xf9, 0xd8, 0x6f, 0xa2, 0x75, 0x9d, 0x45, + 0x08, 0x78, 0xe8, 0x78, 0x1d, 0x45, 0x78, 0x26, 0x5b, 0xe0, 0x1e, 0x45, 0x89, 0xdf, 0xee, 0x3a, + 0x3d, 0x97, 0xe9, 0x23, 0x72, 0xf8, 0x95, 0x0f, 0x96, 0xc3, 0xaf, 0x28, 0x85, 0xcd, 0x60, 0x3c, + 0x91, 0xa9, 0x62, 0x22, 0x9b, 0xc8, 0x24, 0x43, 0xab, 0xc3, 0x34, 0xf5, 0x1d, 0x74, 0xa4, 0x8f, + 0xf4, 0x67, 0xd8, 0x3a, 0x88, 0xe5, 0xf0, 0x7c, 0xc0, 0x15, 0x67, 0xe2, 0xa7, 0x5c, 0x64, 0x8a, + 0xdc, 0x83, 0x26, 0x16, 0x6a, 0xf5, 0x0c, 0xa1, 0xb9, 0x58, 0xb0, 0xdf, 0x30, 0x5c, 0x24, 0x34, + 0x17, 0xed, 0xb1, 0x6a, 0x8f, 0x19, 0x42, 0x73, 0xc3, 0x38, 0x1a, 0x9a, 0x6a, 0x3d, 0x66, 0x08, + 0x5d, 0xc7, 0xd3, 0x48, 0x5c, 0xda, 0x12, 0xf1, 0x4c, 0x03, 0xd8, 0xae, 0xc5, 0xb7, 0x69, 0xee, + 0x40, 0x8b, 0xc9, 0xcb, 0x60, 0x90, 0xf9, 0x4e, 0xd7, 0xed, 0x79, 0xcc, 0x52, 0xd8, 0x48, 0x19, + 0xe7, 0xe3, 0x44, 0x8b, 0x1a, 0x28, 0xaa, 0x18, 0xf4, 0x3e, 0x34, 0xb1, 0xab, 0xba, 0xca, 0xca, + 0x56, 0x1f, 0xe9, 0x2f, 0x0e, 0xb4, 0x8f, 0xf8, 0x15, 0xa6, 0x91, 0x91, 0x27, 0xb0, 0x1e, 0x2a, + 0x9e, 0x8c, 0x78, 0x3a, 0x42, 0xa5, 0x8d, 0x47, 0xef, 0xef, 0x15, 0x53, 0xde, 0x2b, 0xd5, 0xf6, + 0x0a, 0x9d, 0xc3, 0x44, 0xa5, 0x53, 0x56, 0x9a, 0xbc, 0xf3, 0x05, 0x74, 0x6e, 0x88, 0x74, 0xbc, + 0x73, 0x31, 0x2d, 0xba, 0x7a, 0x2e, 0xa6, 0xba, 0xfe, 0x0b, 0x1e, 0xe7, 0xa6, 0x57, 0x1e, 0x33, + 0xc4, 0xe7, 0x8d, 0x4f, 0x1d, 0xba, 0x0f, 0xa4, 0x9f, 0x0a, 0xae, 0x04, 0x06, 0x39, 0x12, 0x59, + 0xc6, 0xcf, 0xc4, 0xf2, 0x8e, 0x9b, 0x2e, 0x36, 0x6a, 0x5d, 0xa4, 0x1f, 0x03, 0x19, 0x88, 0x58, + 0x28, 0x61, 0xc1, 0xf8, 0x12, 0x0f, 0x34, 0x2c, 0xa2, 0xdd, 0xae, 0x4b, 0x3e, 0x04, 0x4f, 0x63, + 0x19, 0x83, 0x6d, 0x3c, 0xba, 0x5b, 0x75, 0xa4, 0x04, 0x3d, 0x43, 0x05, 0x1a, 0x15, 0x4e, 0x2d, + 0xfe, 0x6f, 0x29, 0x61, 0x01, 0x68, 0x8a, 0x50, 0xee, 0x6c, 0xa8, 0x72, 0xa3, 0x6c, 0xa8, 0xfd, + 0xa2, 0xd6, 0x55, 0x43, 0xd1, 0xb3, 0x32, 0xd9, 0x48, 0xc4, 0xa3, 0x55, 0x92, 0xfd, 0x00, 0x9a, + 0x68, 0x6b, 0xb3, 0xbd, 0x53, 0xcb, 0x56, 0xb3, 0x99, 0x91, 0xd2, 0xa7, 0x65, 0xaa, 0xab, 0x06, + 0xba, 0x57, 0x0f, 0xd4, 0x2e, 0xfc, 0xfe, 0x60, 0x75, 0xf5, 0xf6, 0x1c, 0x6b, 0x1b, 0xe3, 0x09, + 0xcf, 0xcb, 0x67, 0x36, 0xd3, 0x48, 0xed, 0x5b, 0xaf, 0x5b, 0xe6, 0xbb, 0x5d, 0x57, 0xfb, 0x46, + 0x82, 0x3e, 0x86, 0x56, 0x38, 0x7c, 0x2e, 0xc6, 0x9c, 0x7c, 0x04, 0x6b, 0x98, 0x9a, 0xc8, 0xec, + 0x46, 0xdc, 0x99, 0x99, 0x3f, 0x2b, 0xe4, 0x74, 0x60, 0x4b, 0x5a, 0x92, 0x50, 0x0b, 0x43, 0x67, + 0xbe, 0x37, 0xeb, 0x06, 0xf9, 0xcc, 0x8a, 0xe9, 0x21, 0xb8, 0xa7, 0x2c, 0xd0, 0x9b, 0x8e, 0x19, + 0x14, 0x5e, 0x2c, 0xa5, 0x7d, 0x7f, 0x2d, 0x33, 0x65, 0x1b, 0x84, 0x67, 0xcd, 0xfb, 0x4e, 0xa6, + 0x0a, 0xdb, 0xd3, 0x61, 0x78, 0xa6, 0x3f, 0x82, 0x77, 0x2c, 0x47, 0x82, 0x6c, 0x42, 0x23, 0x18, + 0x58, 0x1f, 0x8d, 0x60, 0x40, 0xde, 0x43, 0xf7, 0xb6, 0x2f, 0x9d, 0x2a, 0x89, 0x53, 0x16, 0x30, + 0x0c, 0xfc, 0x00, 0x3a, 0x41, 0xd6, 0x97, 0x32, 0x1d, 0x45, 0x09, 0x57, 0x32, 0x45, 0xaf, 0xeb, + 0xec, 0x26, 0x93, 0xee, 0xc3, 0x96, 0x76, 0x1f, 0x2a, 0xae, 0x4a, 0xf4, 0xed, 0x40, 0x4b, 0xf3, + 0xca, 0x70, 0x96, 0xc2, 0x6d, 0xd5, 0x7a, 0xc5, 0x50, 0x91, 0xa0, 0xdf, 0x1a, 0x0f, 0x87, 0x17, + 0x22, 0x51, 0x35, 0x50, 0x20, 0x8d, 0x0e, 0x3a, 0xcc, 0x10, 0x84, 0x9a, 0x52, 0x6c, 0xce, 0x9b, + 0x55, 0xce, 0x9a, 0xcb, 0x50, 0x46, 0x7f, 0x73, 0x00, 0x8a, 0x84, 0xf2, 0xac, 0x34, 0x71, 0x96, + 0x9b, 0x90, 0x4f, 0x6a, 0x37, 0xdf, 0x3c, 0x4e, 0x4a, 0x11, 0xab, 0xdd, 0x8f, 0xbd, 0x02, 0x16, + 0x16, 0xf2, 0x5b, 0x95, 0xbe, 0xe1, 0xdb, 0x31, 0xe9, 0xab, 0xa0, 0xd3, 0x8f, 0xf3, 0x4c, 0x89, + 0xd4, 0x66, 0xa4, 0x6f, 0x68, 0xc3, 0x28, 0xfb, 0x53, 0x31, 0x16, 0xb7, 0x88, 0x3c, 0x80, 0xa6, + 0xce, 0xd4, 0x60, 0x73, 0xbe, 0x0c, 0x23, 0xa4, 0xa1, 0xdd, 0x8e, 0x85, 0xb0, 0x2b, 0x5e, 0xc8, + 0xc6, 0xfc, 0x0b, 0xe9, 0xce, 0xbd, 0x90, 0x5e, 0xf5, 0x42, 0x86, 0xb0, 0x6d, 0x6e, 0x07, 0xbd, + 0x0f, 0xab, 0xec, 0x6c, 0xf1, 0xa4, 0xb9, 0xb5, 0x27, 0x2d, 0x84, 0x6d, 0x73, 0x13, 0xbc, 0x4e, + 0xa7, 0x7f, 0x34, 0x60, 0x9b, 0x89, 0x2c, 0x7a, 0x21, 0x82, 0x24, 0x53, 0x69, 0x3e, 0x54, 0x91, + 0x4c, 0xb4, 0xfd, 0x37, 0xf2, 0x99, 0x6d, 0xb5, 0xcb, 0x0c, 0xf1, 0x2a, 0x48, 0x22, 0x0f, 0x61, + 0x63, 0x16, 0xfd, 0xf3, 0xaa, 0x75, 0x15, 0xf2, 0x10, 0xd6, 0x42, 0x99, 0xa7, 0xc3, 0x72, 0xb7, + 0x77, 0x2a, 0x6d, 0x93, 0x99, 0x11, 0xb3, 0x42, 0xad, 0x86, 0xa3, 0xe6, 0xcb, 0x71, 0x44, 0x9e, + 0xcc, 0xe0, 0xc8, 0x6f, 0xa1, 0xc1, 0xdb, 0x95, 0xc1, 0x0d, 0x31, 0xbb, 0xa9, 0x4d, 0x7f, 0x75, + 0xe0, 0xcd, 0x7a, 0x0a, 0xaf, 0xb4, 0x18, 0xe5, 0x44, 0x1a, 0x0b, 0x27, 0xe2, 0x2e, 0x9a, 0x88, + 0x57, 0x4d, 0xa4, 0x7a, 0x9d, 0x9b, 0xf5, 0xd7, 0xf9, 0x1c, 0xee, 0xcf, 0x8d, 0xa9, 0x2f, 0xc7, + 0x13, 0x8d, 0x87, 0xff, 0x31, 0x2e, 0x7d, 0x65, 0xa4, 0xa9, 0x1d, 0x54, 0x9b, 0x19, 0x82, 0x7e, + 0x06, 0x6f, 0x85, 0x42, 0xd5, 0x86, 0x54, 0xa0, 0xad, 0x0b, 0xee, 0xb1, 0xb8, 0x5c, 0x52, 0xbe, + 0x16, 0xd1, 0x2f, 0xc1, 0x3f, 0x9d, 0x8c, 0xb8, 0x12, 0x2b, 0x59, 0x1f, 0xc0, 0xfa, 0x89, 0x9c, + 0xc8, 0x58, 0x9e, 0x4d, 0x6f, 0x59, 0x79, 0x1f, 0xd6, 0xcc, 0xfd, 0x68, 0x3e, 0x6c, 0x6d, 0x56, + 0x90, 0xf4, 0xae, 0x06, 0xf4, 0x90, 0xc7, 0xc3, 0x3c, 0xd6, 0x69, 0xe8, 0x9f, 0x5b, 0x76, 0xb0, + 0xf5, 0xd7, 0xf5, 0xae, 0xf3, 0xf7, 0xf5, 0xae, 0xf3, 0xcf, 0xf5, 0xae, 0xf3, 0xfb, 0xbf, 0xbb, + 0x6f, 0x3c, 0x6b, 0xe1, 0x37, 0xfc, 0xf1, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x71, 0x97, 0x84, + 0xb1, 0x97, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 52e587f4b..b530257ae 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -6,10 +6,13 @@ message IndexMeta { } message FrameMeta { + string Type = 8; string CacheType = 3; uint32 CacheSize = 4; + int64 Min = 9; + int64 Max = 10; string TimeQuantum = 5; - repeated Field Fields = 7; + //repeated Field Fields = 7; } message ImportResponse { diff --git a/internal/public.pb.go b/internal/public.pb.go index 0dab2c831..069f633c0 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,5 +1,6 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. +// Code generated by protoc-gen-gogo. // source: public.proto +// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -27,8 +28,6 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import binary "encoding/binary" - import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -808,8 +807,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) - i += 8 + i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue)))) } return i, nil } @@ -1251,6 +1249,24 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Public(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Public(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2335,8 +2351,15 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } - v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 + v = uint64(dAtA[iNdEx-8]) + v |= uint64(dAtA[iNdEx-7]) << 8 + v |= uint64(dAtA[iNdEx-6]) << 16 + v |= uint64(dAtA[iNdEx-5]) << 24 + v |= uint64(dAtA[iNdEx-4]) << 32 + v |= uint64(dAtA[iNdEx-3]) << 40 + v |= uint64(dAtA[iNdEx-2]) << 48 + v |= uint64(dAtA[iNdEx-1]) << 56 m.FloatValue = float64(math.Float64frombits(v)) default: iNdEx = preIndex From dbb4cdf3909f08320888eb34050f1f0550fb7af1 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 1 Jun 2018 21:03:18 -0500 Subject: [PATCH 08/78] adjust the tests to match the new unexported Field and FrameOptions --- client_test.go | 21 ++-- cluster_test.go | 34 ------ ctl/import_test.go | 5 +- executor_test.go | 223 ++++++++++++++++++++------------------ frame.go | 37 +++++-- frame_internal_test.go | 149 +++++++++++++++++++++++++ frame_test.go | 190 +++++--------------------------- handler_test.go | 240 ----------------------------------------- index_test.go | 156 +++++++++++++-------------- test/frame.go | 8 +- 10 files changed, 423 insertions(+), 640 deletions(-) create mode 100644 frame_internal_test.go diff --git a/client_test.go b/client_test.go index 6e5f2dc41..898869f2b 100644 --- a/client_test.go +++ b/client_test.go @@ -244,16 +244,17 @@ func TestClient_ImportValue(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - fld := pilosa.Field{ - Name: "fld", - Type: pilosa.FieldTypeInt, + fldName := "f" + + fo := pilosa.FrameOptions{ + Type: pilosa.FrameTypeInt, Min: -100, Max: 100, } // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}}) + frame, err := index.CreateFrameIfNotExists(fldName, fo) if err != nil { t.Fatal(err) } @@ -266,7 +267,7 @@ func TestClient_ImportValue(t *testing.T) { // Send import request. c := test.MustNewClient(s.Host(), defaultClient) - if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{ + if err := c.ImportValue(context.Background(), "i", "f", fldName, 0, []pilosa.FieldValue{ {ColumnID: 1, Value: -10}, {ColumnID: 2, Value: 20}, {ColumnID: 3, Value: 40}, @@ -275,7 +276,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Sum. - sum, cnt, err := frame.FieldSum(nil, fld.Name) + sum, cnt, err := frame.FieldSum(nil, fldName) if err != nil { t.Fatal(err) } @@ -284,7 +285,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Min. - min, cnt, err := frame.FieldMin(nil, fld.Name) + min, cnt, err := frame.FieldMin(nil, fldName) if err != nil { t.Fatal(err) } @@ -293,11 +294,11 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Min with Filter. - filter, err := frame.FieldRange(fld.Name, pql.GT, 40) + filter, err := frame.FieldRange(fldName, pql.GT, 40) if err != nil { t.Fatal(err) } - min, cnt, err = frame.FieldMin(filter, fld.Name) + min, cnt, err = frame.FieldMin(filter, fldName) if err != nil { t.Fatal(err) } @@ -306,7 +307,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Max. - max, cnt, err := frame.FieldMax(nil, fld.Name) + max, cnt, err := frame.FieldMax(nil, fldName) if err != nil { t.Fatal(err) } diff --git a/cluster_test.go b/cluster_test.go index a2570c41a..99b322a0e 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -422,24 +422,6 @@ func TestCluster_ResizeStates(t *testing.T) { tc.SetBit("i", "f", "standard", 1, 101, nil) tc.SetBit("i", "f", "standard", 1, 1300000, nil) - // Add Field Data to node0. - if err := tc.CreateFrame("i", "fields", FrameOptions{ - Fields: []*oField{ - { - Name: "fld0", - Type: FieldTypeInt, - Min: -100, - Max: 100, - }, - }, - }); err != nil { - t.Fatal(err) - } - tc.SetFieldValue("i", "fields", 1, "fld0", -10) - tc.SetFieldValue("i", "fields", 1, "fld0", 10) - tc.SetFieldValue("i", "fields", 1300000, "fld0", -99) - tc.SetFieldValue("i", "fields", 1300000, "fld0", 99) - // Before starting the resize, get the CheckSum to use for // comparison later. node0Frame := node0.Holder.Frame("i", "f") @@ -447,11 +429,6 @@ func TestCluster_ResizeStates(t *testing.T) { node0Fragment := node0View.Fragment(1) node0Checksum := node0Fragment.Checksum() - node0Frame = node0.Holder.Frame("i", "fields") - node0View = node0Frame.View("field_fld0") - node0Fragment = node0View.Fragment(1) - node0ChecksumFld := node0Fragment.Checksum() - // AddNode needs to block until the resize process has completed. tc.AddNode(false) node1 := tc.Clusters[1] @@ -485,17 +462,6 @@ func TestCluster_ResizeStates(t *testing.T) { t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) } - // Values - // Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0. - node1Frame = node1.Holder.Frame("i", "fields") - node1View = node1Frame.View("field_fld0") - node1Fragment = node1View.Fragment(1) - - // Ensure checksums are the same. - if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0ChecksumFld) { - t.Fatalf("expected checksum to match: %x - %x", chksum, node0ChecksumFld) - } - // Close TestCluster. if err := tc.Close(); err != nil { t.Fatal(err) diff --git a/ctl/import_test.go b/ctl/import_test.go index a11770bd8..7c6c02721 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -82,6 +82,7 @@ func TestImportCommand_Run(t *testing.T) { } } +// TODO: revisit this test once Frame is renamed Field // Ensure that the ImportValue path runs (note: we have specified a value // for cm.Field.) func TestImportCommand_RunValue(t *testing.T) { @@ -107,11 +108,11 @@ func TestImportCommand_RunValue(t *testing.T) { cm.Host = s.Host() http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) cm.Index = "i" cm.Frame = "f" - cm.Field = "foo" + cm.Field = "f" cm.Paths = []string{file.Name()} err = cm.Run(ctx) if err != nil { diff --git a/executor_test.go b/executor_test.go index 0807228ac..fbce4d52d 100644 --- a/executor_test.go +++ b/executor_test.go @@ -272,10 +272,9 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { // Create frames. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 50}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 1, Max: 2}, - }, + Type: pilosa.FrameTypeInt, + Min: 0, + Max: 50, }); err != nil { t.Fatal(err) } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil { @@ -284,14 +283,14 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { // Set field values. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, f=25)`), nil, nil); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, frame=f, field0=10)`), nil, nil); err != nil { + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, frame=f, f=10)`), nil, nil); err != nil { t.Fatal(err) } f := hldr.Frame("i", "f") - if value, exists, err := f.FieldValue(10, "field0"); err != nil { + if value, exists, err := f.FieldValue(10, "f"); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") @@ -299,15 +298,7 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { t.Fatalf("unexpected value: %v", value) } - if value, exists, err := f.FieldValue(10, "field1"); err != nil { - t.Fatal(err) - } else if !exists { - t.Fatal("expected value to exist") - } else if value != 2 { - t.Fatalf("unexpected value: %v", value) - } - - if value, exists, err := f.FieldValue(100, "field0"); err != nil { + if value, exists, err := f.FieldValue(100, "f"); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") @@ -321,37 +312,37 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}, - }, + Type: pilosa.FrameTypeInt, + Min: 0, + Max: 100, }); err != nil { t.Fatal(err) } t.Run("ErrFrameRequired", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, f=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnFieldRequired", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, f=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnFieldValue", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, f=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidFieldValueType", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, f="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { t.Fatalf("unexpected error: %s", err) } }) @@ -588,29 +579,33 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } + if _, err := idx.CreateFrame("x", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100}, - }, + Type: pilosa.FrameTypeInt, + Min: -10, + Max: 100, }); err != nil { t.Fatal(err) } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=3) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetBit(frame=f, row=1, col=1) - SetBit(frame=f, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(frame=x, row=0, col=0) + SetBit(frame=x, row=0, col=3) + SetBit(frame=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=x, row=1, col=1) + SetBit(frame=x, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=20, col=0) - SetFieldValue(frame=f, foo=-5, col=1) - SetFieldValue(frame=f, foo=-5, col=2) - SetFieldValue(frame=f, foo=10, col=3) - SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=f, f=20, col=0) + SetFieldValue(frame=f, f=-5, col=1) + SetFieldValue(frame=f, f=-5, col=2) + SetFieldValue(frame=f, f=10, col=3) + SetFieldValue(frame=f, f=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, f=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, f=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, f=60, col=`+strconv.Itoa(SliceWidth+1)+`) `), nil, nil); err != nil { t.Fatal(err) } @@ -622,16 +617,16 @@ func TestExecutor_Execute_MinMax(t *testing.T) { cnt int64 }{ {filter: ``, exp: -5, cnt: 2}, - {filter: `Bitmap(frame=f, row=0)`, exp: 10, cnt: 1}, - {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, - {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + {filter: `Bitmap(frame=x, row=0)`, exp: 10, cnt: 1}, + {filter: `Bitmap(frame=x, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(frame=x, row=2)`, exp: 40, cnt: 1}, } for i, tt := range tests { var pql string if tt.filter == "" { - pql = `Min(frame=f, field=foo)` + pql = `Min(frame=f, field=f)` } else { - pql = fmt.Sprintf(`Min(%s, frame=f, field=foo)`, tt.filter) + pql = fmt.Sprintf(`Min(%s, frame=f, field=f)`, tt.filter) } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) @@ -648,16 +643,16 @@ func TestExecutor_Execute_MinMax(t *testing.T) { cnt int64 }{ {filter: ``, exp: 60, cnt: 1}, - {filter: `Bitmap(frame=f, row=0)`, exp: 60, cnt: 1}, - {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, - {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + {filter: `Bitmap(frame=x, row=0)`, exp: 60, cnt: 1}, + {filter: `Bitmap(frame=x, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(frame=x, row=2)`, exp: 40, cnt: 1}, } for i, tt := range tests { var pql string if tt.filter == "" { - pql = `Max(frame=f, field=foo)` + pql = `Max(frame=f, field=f)` } else { - pql = fmt.Sprintf(`Max(%s, frame=f, field=foo)`, tt.filter) + pql = fmt.Sprintf(`Max(%s, frame=f, field=f)`, tt.filter) } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) @@ -679,39 +674,51 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, - {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, - }, + if _, err := idx.CreateFrame("x", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("foo", pilosa.FrameOptions{ + Type: pilosa.FrameTypeInt, + Min: 10, + Max: 100, + }); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{ + Type: pilosa.FrameTypeInt, + Min: 0, + Max: 100000, }); err != nil { t.Fatal(err) } if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, - }, + Type: pilosa.FrameTypeInt, + Min: 0, + Max: 1000, }); err != nil { t.Fatal(err) } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=x, row=0, col=0) + SetBit(frame=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=f, foo=20, bar=2000, col=0) - SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, col=0) + SetFieldValue(frame=foo, foo=20, col=0) + SetFieldValue(frame=bar, bar=2000, col=0) + SetFieldValue(frame=foo, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=foo, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=foo, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=foo, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, other=1000, col=0) `), nil, nil); err != nil { t.Fatal(err) } t.Run("NoFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=foo, field=foo)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -719,7 +726,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=x, row=0), frame=foo, field=foo)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -738,6 +745,7 @@ func TestExecutor_Execute_Range(t *testing.T) { // Create frame. if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ + Type: pilosa.FrameTypeTime, TimeQuantum: pilosa.TimeQuantum("YMDH"), }); err != nil { t.Fatal(err) @@ -766,7 +774,6 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Fatalf("unexpected columns: %+v", columns) } }) - } // Ensure a Range(field) query can be executed. @@ -780,27 +787,38 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, - {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, - }, + if _, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("foo", pilosa.FrameOptions{ + Type: pilosa.FrameTypeInt, + Min: 10, + Max: 100, + }); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{ + Type: pilosa.FrameTypeInt, + Min: 0, + Max: 100000, }); err != nil { t.Fatal(err) } if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, - }, + Type: pilosa.FrameTypeInt, + Min: 0, + Max: 1000, }); err != nil { t.Fatal(err) } if _, err := idx.CreateFrame("edge", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: -100, Max: 100}, - }, + Type: pilosa.FrameTypeInt, + Min: -100, + Max: 100, }); err != nil { t.Fatal(err) } @@ -809,20 +827,21 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { SetBit(frame=f, row=0, col=0) SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=f, foo=20, bar=2000, col=50) - SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, col=0) - SetFieldValue(frame=edge, foo=100, col=0) - SetFieldValue(frame=edge, foo=-100, col=1) + SetFieldValue(frame=foo, foo=20, col=50) + SetFieldValue(frame=bar, bar=2000, col=50) + SetFieldValue(frame=foo, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=foo, foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=foo, foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=foo, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, other=1000, col=0) + SetFieldValue(frame=edge, edge=100, col=0) + SetFieldValue(frame=edge, edge=-100, col=1) `), nil, nil); err != nil { t.Fatal(err) } t.Run("EQ", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo == 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -831,19 +850,19 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("NEQ", func(t *testing.T) { // NEQ null - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != null)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, other != null)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo != 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo != 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != -20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, other != -20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -852,7 +871,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("LT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo < 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -860,7 +879,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("LTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo <= 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -868,7 +887,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("GT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo > 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -876,7 +895,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("GTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo >= 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -884,7 +903,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("BETWEEN", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [1, 1000])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, other >< [1, 1000])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -893,7 +912,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { // Ensure that the FieldNotNull code path gets run. t.Run("FieldNotNull", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [0, 1000])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, other >< [0, 1000])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -901,7 +920,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("BelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo == 0)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -909,7 +928,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("AboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo == 200)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -917,7 +936,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("LTAboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo < 200)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, edge < 200)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) @@ -925,7 +944,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo > -200)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, edge > -200)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) @@ -939,7 +958,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("ErrFieldNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { t.Fatal(err) } }) diff --git a/frame.go b/frame.go index e0765096e..35d417136 100644 --- a/frame.go +++ b/frame.go @@ -69,14 +69,25 @@ type Frame struct { Logger Logger } +// FrameOption is a functional option type for pilosa.Frame. +type FrameOption func(f *Frame) error + +// TODO: break these out into separate Options (not a FrameOptions object) +func OptFrameFrameOptions(o FrameOptions) FrameOption { + return func(f *Frame) error { + f.options = o + return nil + } +} + // NewFrame returns a new instance of frame. -func NewFrame(path, index, name string) (*Frame, error) { +func NewFrame(path, index, name string, opts ...FrameOption) (*Frame, error) { err := ValidateName(name) if err != nil { return nil, err } - return &Frame{ + f := &Frame{ path: path, index: index, name: name, @@ -95,7 +106,16 @@ func NewFrame(path, index, name string) (*Frame, error) { }, Logger: NopLogger, - }, nil + } + + for _, opt := range opts { + err := opt(f) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + + return f, nil } // Name returns the name the frame was initialized with. @@ -299,7 +319,7 @@ func (f *Frame) applyOptions(opt FrameOptions) error { f.options.TimeQuantum = "" case FrameTypeInt: f.options.Type = opt.Type - f.options.CacheType = "" + f.options.CacheType = CacheTypeNone f.options.CacheSize = 0 f.options.Min = opt.Min f.options.Max = opt.Max @@ -321,7 +341,7 @@ func (f *Frame) applyOptions(opt FrameOptions) error { } case FrameTypeTime: f.options.Type = opt.Type - f.options.CacheType = "" + f.options.CacheType = CacheTypeNone f.options.CacheSize = 0 f.options.Min = 0 f.options.Max = 0 @@ -657,7 +677,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change // Clear non-time bit. if v, err := view.ClearBit(rowID, colID); err != nil { - return changed, errors.Wrap(err, "setting on view") + return changed, errors.Wrap(err, "clearing on view") } else if v { changed = v } @@ -675,7 +695,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change } if c, err := view.ClearBit(rowID, colID); err != nil { - return changed, errors.Wrapf(err, "setting on view %s", subname) + return changed, errors.Wrapf(err, "clearing on view %s", subname) } else if c { changed = true } @@ -995,7 +1015,7 @@ type FrameOptions struct { CacheSize uint32 `json:"cacheSize,omitempty"` Min int64 `json:"min,omitempty"` Max int64 `json:"max,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` // TODO travis: rename this Quantum? + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` // TODO: rename this Quantum? } // Validate ensures that FrameOption values are valid. @@ -1064,6 +1084,7 @@ func IsValidFieldType(v string) bool { } } +// TODO: finish unexporting this. also, rename it. // oField represents a range field on a frame. type oField struct { Name string `json:"name,omitempty"` diff --git a/frame_internal_test.go b/frame_internal_test.go new file mode 100644 index 000000000..f51ad95e9 --- /dev/null +++ b/frame_internal_test.go @@ -0,0 +1,149 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "reflect" + "testing" + + "github.com/pilosa/pilosa/pql" +) + +// Ensure a field can adjust to its baseValue. +func TestField_BaseValue(t *testing.T) { + f0 := &oField{ + Name: "f0", + Type: FieldTypeInt, + Min: -100, + Max: 900, + } + f1 := &oField{ + Name: "f1", + Type: FieldTypeInt, + Min: 0, + Max: 1000, + } + + f2 := &oField{ + Name: "f2", + Type: FieldTypeInt, + Min: 100, + Max: 1100, + } + + t.Run("Normal Condition", func(t *testing.T) { + + for _, tt := range []struct { + f *oField + op pql.Token + val int64 + expBaseValue uint64 + expOutOfRange bool + }{ + // LT + {f0, pql.LT, 5, 105, false}, + {f0, pql.LT, -8, 92, false}, + {f0, pql.LT, -108, 0, true}, + {f0, pql.LT, 1005, 1000, false}, + {f0, pql.LT, 0, 100, false}, + + {f1, pql.LT, 5, 5, false}, + {f1, pql.LT, -8, 0, true}, + {f1, pql.LT, 1005, 1000, false}, + {f1, pql.LT, 0, 0, false}, + + {f2, pql.LT, 5, 0, true}, + {f2, pql.LT, -8, 0, true}, + {f2, pql.LT, 105, 5, false}, + {f2, pql.LT, 1105, 1000, false}, + + // GT + {f0, pql.GT, -105, 0, false}, + {f0, pql.GT, 5, 105, false}, + {f0, pql.GT, 905, 0, true}, + {f0, pql.GT, 0, 100, false}, + + {f1, pql.GT, 5, 5, false}, + {f1, pql.GT, -8, 0, false}, + {f1, pql.GT, 1005, 0, true}, + {f1, pql.GT, 0, 0, false}, + + {f2, pql.GT, 5, 0, false}, + {f2, pql.GT, -8, 0, false}, + {f2, pql.GT, 105, 5, false}, + {f2, pql.GT, 1105, 0, true}, + + // EQ + {f0, pql.EQ, -105, 0, true}, + {f0, pql.EQ, 5, 105, false}, + {f0, pql.EQ, 905, 0, true}, + {f0, pql.EQ, 0, 100, false}, + + {f1, pql.EQ, 5, 5, false}, + {f1, pql.EQ, -8, 0, true}, + {f1, pql.EQ, 1005, 0, true}, + {f1, pql.EQ, 0, 0, false}, + + {f2, pql.EQ, 5, 0, true}, + {f2, pql.EQ, -8, 0, true}, + {f2, pql.EQ, 105, 5, false}, + {f2, pql.EQ, 1105, 0, true}, + } { + bv, oor := tt.f.BaseValue(tt.op, tt.val) + if oor != tt.expOutOfRange { + t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor) + } else if !reflect.DeepEqual(bv, tt.expBaseValue) { + t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv) + } + } + }) + + t.Run("Betwween Condition", func(t *testing.T) { + for _, tt := range []struct { + f *oField + predMin int64 + predMax int64 + expBaseValueMin uint64 + expBaseValueMax uint64 + expOutOfRange bool + }{ + + {f0, -205, -105, 0, 0, true}, + {f0, -105, 80, 0, 180, false}, + {f0, 5, 20, 105, 120, false}, + {f0, 20, 1005, 120, 1000, false}, + {f0, 1005, 2000, 0, 0, true}, + + {f1, -105, -5, 0, 0, true}, + {f1, -5, 20, 0, 20, false}, + {f1, 5, 20, 5, 20, false}, + {f1, 20, 1005, 20, 1000, false}, + {f1, 1005, 2000, 0, 0, true}, + + {f2, 5, 95, 0, 0, true}, + {f2, 95, 120, 0, 20, false}, + {f2, 105, 120, 5, 20, false}, + {f2, 120, 1105, 20, 1000, false}, + {f2, 1105, 2000, 0, 0, true}, + } { + min, max, oor := tt.f.BaseValueBetween(tt.predMin, tt.predMax) + if oor != tt.expOutOfRange { + t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor) + } else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) { + t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max) + } + } + }) +} diff --git a/frame_test.go b/frame_test.go index 78014bbce..8ad9b020a 100644 --- a/frame_test.go +++ b/frame_test.go @@ -16,11 +16,9 @@ package pilosa_test import ( "io/ioutil" - "reflect" "testing" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" ) @@ -52,7 +50,10 @@ func TestFrame_CreateViewIfNotExists(t *testing.T) { // Ensure frame can set its time quantum. func TestFrame_SetTimeQuantum(t *testing.T) { - f := test.MustOpenFrame() + fo := pilosa.FrameOptions{ + Type: "time", + } + f := test.MustOpenFrame(pilosa.OptFrameFrameOptions(fo)) defer f.Close() // Set & retrieve time quantum. @@ -77,31 +78,23 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25}, - }, + Type: pilosa.FrameTypeInt, + Min: 0, + Max: 30, }) if err != nil { t.Fatal(err) } - // Set value on first field. - if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - // Set value on same column but different field. - if changed, err := f.SetFieldValue(100, "field1", 25); err != nil { + // Set value on field. + if changed, err := f.SetFieldValue(100, "f", 21); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.FieldValue(100, "field0"); err != nil { + if value, exists, err := f.FieldValue(100, "f"); err != nil { t.Fatal(err) } else if value != 21 { t.Fatalf("unexpected value: %d", value) @@ -110,7 +103,7 @@ func TestFrame_SetFieldValue(t *testing.T) { } // Setting value should return no change. - if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { + if changed, err := f.SetFieldValue(100, "f", 21); err != nil { t.Fatal(err) } else if changed { t.Fatal("expected no change") @@ -122,30 +115,30 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, - }, + Type: pilosa.FrameTypeInt, + Min: 0, + Max: 30, }) if err != nil { t.Fatal(err) } // Set value. - if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { + if changed, err := f.SetFieldValue(100, "f", 21); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Set different value. - if changed, err := f.SetFieldValue(100, "field0", 23); err != nil { + if changed, err := f.SetFieldValue(100, "f", 23); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.FieldValue(100, "field0"); err != nil { + if value, exists, err := f.FieldValue(100, "f"); err != nil { t.Fatal(err) } else if value != 23 { t.Fatalf("unexpected value: %d", value) @@ -159,9 +152,9 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, - }, + Type: pilosa.FrameTypeInt, + Min: 0, + Max: 30, }) if err != nil { t.Fatal(err) @@ -178,16 +171,16 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, - }, + Type: pilosa.FrameTypeInt, + Min: 20, + Max: 30, }) if err != nil { t.Fatal(err) } // Set value. - if _, err := f.SetFieldValue(100, "field0", 15); err != pilosa.ErrFieldValueTooLow { + if _, err := f.SetFieldValue(100, "f", 15); err != pilosa.ErrFieldValueTooLow { t.Fatalf("unexpected error: %s", err) } }) @@ -197,16 +190,16 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, - }, + Type: pilosa.FrameTypeInt, + Min: 20, + Max: 30, }) if err != nil { t.Fatal(err) } // Set value. - if _, err := f.SetFieldValue(100, "field0", 31); err != pilosa.ErrFieldValueTooHigh { + if _, err := f.SetFieldValue(100, "f", 31); err != pilosa.ErrFieldValueTooHigh { t.Fatalf("unexpected error: %s", err) } }) @@ -294,130 +287,3 @@ func TestFrame_DeleteView(t *testing.T) { t.Fatal("failed to create new view") } } - -// Ensure a field can adjust to its baseValue. -func TestField_BaseValue(t *testing.T) { - f0 := &pilosa.Field{ - Name: "f0", - Type: pilosa.FieldTypeInt, - Min: -100, - Max: 900, - } - f1 := &pilosa.Field{ - Name: "f1", - Type: pilosa.FieldTypeInt, - Min: 0, - Max: 1000, - } - - f2 := &pilosa.Field{ - Name: "f2", - Type: pilosa.FieldTypeInt, - Min: 100, - Max: 1100, - } - - t.Run("Normal Condition", func(t *testing.T) { - - for _, tt := range []struct { - f *pilosa.Field - op pql.Token - val int64 - expBaseValue uint64 - expOutOfRange bool - }{ - // LT - {f0, pql.LT, 5, 105, false}, - {f0, pql.LT, -8, 92, false}, - {f0, pql.LT, -108, 0, true}, - {f0, pql.LT, 1005, 1000, false}, - {f0, pql.LT, 0, 100, false}, - - {f1, pql.LT, 5, 5, false}, - {f1, pql.LT, -8, 0, true}, - {f1, pql.LT, 1005, 1000, false}, - {f1, pql.LT, 0, 0, false}, - - {f2, pql.LT, 5, 0, true}, - {f2, pql.LT, -8, 0, true}, - {f2, pql.LT, 105, 5, false}, - {f2, pql.LT, 1105, 1000, false}, - - // GT - {f0, pql.GT, -105, 0, false}, - {f0, pql.GT, 5, 105, false}, - {f0, pql.GT, 905, 0, true}, - {f0, pql.GT, 0, 100, false}, - - {f1, pql.GT, 5, 5, false}, - {f1, pql.GT, -8, 0, false}, - {f1, pql.GT, 1005, 0, true}, - {f1, pql.GT, 0, 0, false}, - - {f2, pql.GT, 5, 0, false}, - {f2, pql.GT, -8, 0, false}, - {f2, pql.GT, 105, 5, false}, - {f2, pql.GT, 1105, 0, true}, - - // EQ - {f0, pql.EQ, -105, 0, true}, - {f0, pql.EQ, 5, 105, false}, - {f0, pql.EQ, 905, 0, true}, - {f0, pql.EQ, 0, 100, false}, - - {f1, pql.EQ, 5, 5, false}, - {f1, pql.EQ, -8, 0, true}, - {f1, pql.EQ, 1005, 0, true}, - {f1, pql.EQ, 0, 0, false}, - - {f2, pql.EQ, 5, 0, true}, - {f2, pql.EQ, -8, 0, true}, - {f2, pql.EQ, 105, 5, false}, - {f2, pql.EQ, 1105, 0, true}, - } { - bv, oor := tt.f.BaseValue(tt.op, tt.val) - if oor != tt.expOutOfRange { - t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor) - } else if !reflect.DeepEqual(bv, tt.expBaseValue) { - t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv) - } - } - }) - - t.Run("Betwween Condition", func(t *testing.T) { - for _, tt := range []struct { - f *pilosa.Field - predMin int64 - predMax int64 - expBaseValueMin uint64 - expBaseValueMax uint64 - expOutOfRange bool - }{ - - {f0, -205, -105, 0, 0, true}, - {f0, -105, 80, 0, 180, false}, - {f0, 5, 20, 105, 120, false}, - {f0, 20, 1005, 120, 1000, false}, - {f0, 1005, 2000, 0, 0, true}, - - {f1, -105, -5, 0, 0, true}, - {f1, -5, 20, 0, 20, false}, - {f1, 5, 20, 5, 20, false}, - {f1, 20, 1005, 20, 1000, false}, - {f1, 1005, 2000, 0, 0, true}, - - {f2, 5, 95, 0, 0, true}, - {f2, 95, 120, 0, 20, false}, - {f2, 105, 120, 5, 20, false}, - {f2, 120, 1105, 20, 1000, false}, - {f2, 1105, 2000, 0, 0, true}, - } { - min, max, oor := tt.f.BaseValueBetween(tt.predMin, tt.predMax) - if oor != tt.expOutOfRange { - t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor) - } else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) { - t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max) - } - } - }) -} diff --git a/handler_test.go b/handler_test.go index a7ba8fa85..30bba9254 100644 --- a/handler_test.go +++ b/handler_test.go @@ -17,7 +17,6 @@ package pilosa_test import ( "bytes" "context" - "encoding/json" "errors" "fmt" "io" @@ -817,245 +816,6 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { } } -// Ensure the handler can create a new field on an existing frame. -func TestHandler_Frame_AddField(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - t.Run("OK", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"int","min":100,"max":200}`), - ) - if err != nil { - t.Fatal(err) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - - if field := f.Field("x"); !reflect.DeepEqual(field, &pilosa.Field{Name: "x", Type: "int", Min: 100, Max: 200}) { - t.Fatalf("unexpected field: %#v", field) - } - }) - - t.Run("ErrInvalidFieldType", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"bad_type","min":100,"max":200}`), - ) - if err != nil { - t.Fatal(err) - } else if body := MustReadAll(resp.Body); string(body) != `creating field: validating field: invalid field type`+"\n" { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) - - t.Run("ErrInvalidFieldRange", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"int","min":200,"max":100}`), - ) - if err != nil { - t.Fatal(err) - } else if body := MustReadAll(resp.Body); string(body) != `creating field: validating field: invalid field range`+"\n" { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) - - t.Run("ErrFieldAlreadyExists", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}}, - }); err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"int","min":0,"max":100}`), - ) - if err != nil { - t.Fatal(err) - } else if body := MustReadAll(resp.Body); string(body) != `creating field: field already exists`+"\n" { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) -} - -// Ensure the handler can delete existing fields. -func TestHandler_Frame_DeleteField(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - t.Run("OK", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { - t.Fatal(err) - } - - req, err := http.NewRequest("DELETE", s.URL+"/index/i/frame/f/field/x", nil) - if err != nil { - t.Fatal(err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - - if field := f.Field("x"); field != nil { - t.Fatalf("expected nil field, got: %#v", field) - } - }) - - t.Run("ErrFieldNotFound", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { - t.Fatal(err) - } - - req, err := http.NewRequest("DELETE", s.URL+"/index/i/frame/f/field/y", nil) - if err != nil { - t.Fatal(err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } else if body, err := ioutil.ReadAll(resp.Body); err != nil { - t.Fatal(err) - } else if strings.TrimSpace(string(body)) != `deleting field: field not found` { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) -} - -func TestHandler_Frame_GetFields(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - t.Run("OK", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 1, Max: 100}); err != nil { - t.Fatal(err) - } - resp, err := http.Get(s.URL + "/index/i/frame/f/fields") - if err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - - var fields FrameFields - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - if err = json.Unmarshal([]byte(body), &fields); err != nil { - t.Fatal(err) - } - field := fields.Fields[0] - if field.Name != "x" { - t.Fatalf("expected field's name: x, actuall name: %v", field.Name) - } else if field.Min != 1 { - t.Fatalf("expected field's min: x, actuall min: %v", field.Min) - } else if field.Max != 100 { - t.Fatalf("expected field's max: x, actuall max: %v", field.Max) - } - - }) - - t.Run("ErrFrameFieldNotAllowed", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - _, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}) - if err != nil { - t.Fatalf("creating frame: %v", err) - } - - resp, err := http.Get(s.URL + "/index/i/frame/f1/fields") - if err != nil { - t.Fatal(err) - } - if err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } else if body, err := ioutil.ReadAll(resp.Body); err != nil { - t.Fatal(err) - } else if strings.TrimSpace(string(body)) == `frame fields not allowed` { - t.Fatalf("shouldn't get frame fields not allowed error: %q", body) - } - }) - -} - -type FrameFields struct { - Fields []pilosa.Field -} - // Ensure the handler can retrieve the version. func TestHandler_Version(t *testing.T) { hldr := test.MustOpenHolder() diff --git a/index_test.go b/index_test.go index 0a42804e2..3a4ee48bc 100644 --- a/index_test.go +++ b/index_test.go @@ -57,7 +57,10 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() // Create frame with explicit quantum. - f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}) + f, err := index.CreateFrame("f", pilosa.FrameOptions{ + Type: pilosa.FrameTypeTime, + TimeQuantum: pilosa.TimeQuantum("YMDH"), + }) if err != nil { t.Fatal(err) } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { @@ -74,103 +77,100 @@ func TestIndex_CreateFrame(t *testing.T) { // Create frame with schema and verify it exists. if f, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }, + Type: pilosa.FrameTypeInt, + Min: 10, + Max: 20, }); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(f.Fields(), []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }) { - t.Fatalf("unexpected fields: %#v", f.Fields()) + } else if !reflect.DeepEqual(f.Type(), pilosa.FrameTypeInt) { + t.Fatalf("unexpected type: %#v", f.Type()) } // Reopen the index & verify the fields are loaded. if err := index.Reopen(); err != nil { t.Fatal(err) - } else if f := index.Frame("f"); !reflect.DeepEqual(f.Fields(), []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }) { - t.Fatalf("unexpected fields after reopen: %#v", f.Fields()) + } else if f := index.Frame("f"); !reflect.DeepEqual(f.Type(), pilosa.FrameTypeInt) { + t.Fatalf("unexpected type after reopen: %#v", f.Type()) } }) - t.Run("ErrRangeCacheAllowed", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + // TODO: These errors don't apply here. Instead, we need these tests + // on frame creation FrameOptions validation. + /* + t.Run("ErrRangeCacheAllowed", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - CacheType: pilosa.CacheTypeRanked, - }); err != nil { - t.Fatal(err) - } - }) + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + CacheType: pilosa.CacheTypeRanked, + }); err != nil { + t.Fatal(err) + } + }) - t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - CacheType: pilosa.CacheTypeNone, - CacheSize: uint32(5), - }); err != nil { - t.Fatal(err) - } - }) + t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + CacheType: pilosa.CacheTypeNone, + CacheSize: uint32(5), + }); err != nil { + t.Fatal(err) + } + }) - t.Run("ErrFrameFieldsAllowed", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrFrameFieldsAllowed", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt}, - }, - }); err != nil { - t.Fatal(err) - } - }) + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt}, + }, + }); err != nil { + t.Fatal(err) + } + }) - t.Run("ErrFieldNameRequired", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrFieldNameRequired", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "", Type: pilosa.FieldTypeInt}, - }, - }); err != pilosa.ErrFieldNameRequired { - t.Fatal(err) - } - }) + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + Fields: []*pilosa.Field{ + {Name: "", Type: pilosa.FieldTypeInt}, + }, + }); err != pilosa.ErrFieldNameRequired { + t.Fatal(err) + } + }) - t.Run("ErrInvalidFieldType", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrInvalidFieldType", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: "bad_type"}, - }, - }); err != pilosa.ErrInvalidFieldType { - t.Fatal(err) - } - }) + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: "bad_type"}, + }, + }); err != pilosa.ErrInvalidFieldType { + t.Fatal(err) + } + }) - t.Run("ErrInvalidFieldRange", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrInvalidFieldRange", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, - }, - }); err != pilosa.ErrInvalidFieldRange { - t.Fatal(err) - } - }) + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, + }, + }); err != pilosa.ErrInvalidFieldRange { + t.Fatal(err) + } + }) + */ }) } diff --git a/test/frame.go b/test/frame.go index e107b7d85..09503a3cf 100644 --- a/test/frame.go +++ b/test/frame.go @@ -29,12 +29,12 @@ type Frame struct { } // NewFrame returns a new instance of Frame d/0. -func NewFrame() *Frame { +func NewFrame(opt ...pilosa.FrameOption) *Frame { path, err := ioutil.TempDir("", "pilosa-frame-") if err != nil { panic(err) } - frame, err := pilosa.NewFrame(path, "i", "f") + frame, err := pilosa.NewFrame(path, "i", "f", opt...) if err != nil { panic(err) } @@ -42,8 +42,8 @@ func NewFrame() *Frame { } // MustOpenFrame returns a new, opened frame at a temporary path. Panic on error. -func MustOpenFrame() *Frame { - f := NewFrame() +func MustOpenFrame(opt ...pilosa.FrameOption) *Frame { + f := NewFrame(opt...) if err := f.Open(); err != nil { panic(err) } From 17c793447379ac3b3b4af4622feacc80685bf59e Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Jun 2018 14:18:40 -0500 Subject: [PATCH 09/78] un-export HasField. remove dead code --- frame.go | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/frame.go b/frame.go index 35d417136..1307df32d 100644 --- a/frame.go +++ b/frame.go @@ -390,8 +390,8 @@ func (f *Frame) Field(name string) *oField { return nil } -// HasField returns true if a field exists on the frame. -func (f *Frame) HasField(name string) bool { +// hasField returns true if a field exists on the frame. +func (f *Frame) hasField(name string) bool { for _, fld := range f.fields { if fld.Name == name { return true @@ -417,7 +417,7 @@ func (f *Frame) CreateField(field *oField) error { func (f *Frame) addField(field *oField) error { if err := ValidateField(field); err != nil { return errors.Wrap(err, "validating field") - } else if f.HasField(field.Name) { + } else if f.hasField(field.Name) { return ErrFieldExists } @@ -1168,28 +1168,6 @@ func ValidateField(f *oField) error { return nil } -func encodeFields(a []*oField) []*internal.Field { - if len(a) == 0 { - return nil - } - other := make([]*internal.Field, len(a)) - for i := range a { - other[i] = encodeField(a[i]) - } - return other -} - -func decodeFields(a []*internal.Field) []*oField { - if len(a) == 0 { - return nil - } - other := make([]*oField, len(a)) - for i := range a { - other[i] = decodeField(a[i]) - } - return other -} - func encodeField(f *oField) *internal.Field { if f == nil { return nil From 90a4d957bdc097de2171bb4f88a62f0213864ae8 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 1 Jun 2018 22:06:45 -0500 Subject: [PATCH 10/78] remove `frame` argument from Frame.SetFieldValue(). Rename it to Frame.SetValue() --- executor.go | 22 ++++++---------- executor_test.go | 65 +++++++++++++++++++++--------------------------- frame.go | 8 +++--- frame_test.go | 18 ++++++-------- test/cluster.go | 3 ++- utils_test.go | 3 ++- 6 files changed, 53 insertions(+), 66 deletions(-) diff --git a/executor.go b/executor.go index 97f42acf1..c8ea9e544 100644 --- a/executor.go +++ b/executor.go @@ -1109,17 +1109,6 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C // executeSetFieldValue executes a SetFieldValue() call. func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { - frameName, ok := c.Args["frame"].(string) - if !ok { - return errors.New("SetFieldValue() frame required") - } - - // Retrieve frame. - frame := e.Holder.Frame(index, frameName) - if frame == nil { - return ErrFrameNotFound - } - // Parse labels. columnID, ok, err := c.UintArg(columnLabel) if err != nil { @@ -1130,23 +1119,28 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq // Copy args and remove reserved fields. args := pql.CopyArgs(c.Args) - delete(args, "frame") // While frame could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. // Also, if we ever need to make ColumnAttrs frame-specific, then having this reserved word prevents backward incompatibility. delete(args, columnLabel) // Set values. for name, value := range args { + // Retrieve frame. + frame := e.Holder.Frame(index, name) + if frame == nil { + return ErrFrameNotFound + } + switch value := value.(type) { case int64: - if _, err := frame.SetFieldValue(columnID, name, value); err != nil { + if _, err := frame.SetValue(columnID, value); err != nil { return err } default: return ErrInvalidFieldValueType } + frame.Stats.Count("SetFieldValue", 1, 1.0) } - frame.Stats.Count("SetFieldValue", 1, 1.0) // Do not forward call if this is already being forwarded. if opt.Remote { diff --git a/executor_test.go b/executor_test.go index fbce4d52d..0b70a9e16 100644 --- a/executor_test.go +++ b/executor_test.go @@ -283,9 +283,9 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { // Set field values. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, f=25)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, f=25)`), nil, nil); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, frame=f, f=10)`), nil, nil); err != nil { + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, f=10)`), nil, nil); err != nil { t.Fatal(err) } @@ -319,30 +319,23 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { t.Fatal(err) } - t.Run("ErrFrameRequired", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, f=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` { - t.Fatalf("unexpected error: %s", err) - } - }) - t.Run("ErrColumnFieldRequired", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, f=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnFieldValue", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, f=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidFieldValueType", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, f="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, f="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { t.Fatalf("unexpected error: %s", err) } }) @@ -598,14 +591,14 @@ func TestExecutor_Execute_MinMax(t *testing.T) { SetBit(frame=x, row=1, col=1) SetBit(frame=x, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, f=20, col=0) - SetFieldValue(frame=f, f=-5, col=1) - SetFieldValue(frame=f, f=-5, col=2) - SetFieldValue(frame=f, f=10, col=3) - SetFieldValue(frame=f, f=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, f=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, f=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, f=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(f=20, col=0) + SetFieldValue(f=-5, col=1) + SetFieldValue(f=-5, col=2) + SetFieldValue(f=10, col=3) + SetFieldValue(f=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(f=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(f=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(f=60, col=`+strconv.Itoa(SliceWidth+1)+`) `), nil, nil); err != nil { t.Fatal(err) } @@ -706,13 +699,13 @@ func TestExecutor_Execute_Sum(t *testing.T) { SetBit(frame=x, row=0, col=0) SetBit(frame=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=foo, foo=20, col=0) - SetFieldValue(frame=bar, bar=2000, col=0) - SetFieldValue(frame=foo, foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=foo, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=foo, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=foo, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, other=1000, col=0) + SetFieldValue(foo=20, col=0) + SetFieldValue(bar=2000, col=0) + SetFieldValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(other=1000, col=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -827,15 +820,15 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { SetBit(frame=f, row=0, col=0) SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=foo, foo=20, col=50) - SetFieldValue(frame=bar, bar=2000, col=50) - SetFieldValue(frame=foo, foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=foo, foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=foo, foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=foo, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, other=1000, col=0) - SetFieldValue(frame=edge, edge=100, col=0) - SetFieldValue(frame=edge, edge=-100, col=1) + SetFieldValue(foo=20, col=50) + SetFieldValue(bar=2000, col=50) + SetFieldValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(other=1000, col=0) + SetFieldValue(edge=100, col=0) + SetFieldValue(edge=-100, col=1) `), nil, nil); err != nil { t.Fatal(err) } diff --git a/frame.go b/frame.go index 1307df32d..f460a1224 100644 --- a/frame.go +++ b/frame.go @@ -726,10 +726,10 @@ func (f *Frame) FieldValue(columnID uint64, name string) (value int64, exists bo return int64(v) + field.Min, true, nil } -// SetFieldValue sets a field value for a column. -func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (changed bool, err error) { +// SetValue sets a field value for a column. +func (f *Frame) SetValue(columnID uint64, value int64) (changed bool, err error) { // Fetch field and validate value. - field := f.Field(name) + field := f.Field(f.name) if field == nil { return false, ErrFieldNotFound } else if value < field.Min { @@ -739,7 +739,7 @@ func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (change } // Fetch target view. - view, err := f.CreateViewIfNotExists(ViewFieldPrefix + name) + view, err := f.CreateViewIfNotExists(ViewFieldPrefix + f.name) if err != nil { return false, errors.Wrap(err, "creating view") } diff --git a/frame_test.go b/frame_test.go index 8ad9b020a..8000886c0 100644 --- a/frame_test.go +++ b/frame_test.go @@ -87,7 +87,7 @@ func TestFrame_SetFieldValue(t *testing.T) { } // Set value on field. - if changed, err := f.SetFieldValue(100, "f", 21); err != nil { + if changed, err := f.SetValue(100, 21); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") @@ -103,7 +103,7 @@ func TestFrame_SetFieldValue(t *testing.T) { } // Setting value should return no change. - if changed, err := f.SetFieldValue(100, "f", 21); err != nil { + if changed, err := f.SetValue(100, 21); err != nil { t.Fatal(err) } else if changed { t.Fatal("expected no change") @@ -124,14 +124,14 @@ func TestFrame_SetFieldValue(t *testing.T) { } // Set value. - if changed, err := f.SetFieldValue(100, "f", 21); err != nil { + if changed, err := f.SetValue(100, 21); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Set different value. - if changed, err := f.SetFieldValue(100, "f", 23); err != nil { + if changed, err := f.SetValue(100, 23); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") @@ -152,16 +152,14 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, - Min: 0, - Max: 30, + Type: pilosa.FrameTypeSet, }) if err != nil { t.Fatal(err) } // Set value. - if _, err := f.SetFieldValue(100, "no_such_field", 21); err != pilosa.ErrFieldNotFound { + if _, err := f.SetValue(100, 21); err != pilosa.ErrFieldNotFound { t.Fatalf("unexpected error: %s", err) } }) @@ -180,7 +178,7 @@ func TestFrame_SetFieldValue(t *testing.T) { } // Set value. - if _, err := f.SetFieldValue(100, "f", 15); err != pilosa.ErrFieldValueTooLow { + if _, err := f.SetValue(100, 15); err != pilosa.ErrFieldValueTooLow { t.Fatalf("unexpected error: %s", err) } }) @@ -199,7 +197,7 @@ func TestFrame_SetFieldValue(t *testing.T) { } // Set value. - if _, err := f.SetFieldValue(100, "f", 31); err != pilosa.ErrFieldValueTooHigh { + if _, err := f.SetValue(100, 31); err != pilosa.ErrFieldValueTooHigh { t.Fatalf("unexpected error: %s", err) } }) diff --git a/test/cluster.go b/test/cluster.go index 77cf496a8..9089089a7 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -150,6 +150,7 @@ func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x * return nil } +// TODO: remove `name` from this function signature func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error { // Determine which node should receive the SetFieldValue. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. @@ -165,7 +166,7 @@ func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name s if f == nil { return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) } - _, err := f.SetFieldValue(columnID, name, value) + _, err := f.SetValue(columnID, value) if err != nil { return err } diff --git a/utils_test.go b/utils_test.go index 1732522d4..26fe06890 100644 --- a/utils_test.go +++ b/utils_test.go @@ -141,6 +141,7 @@ func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, return nil } +// TODO: remove `name` from this function signature func (t *ClusterCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error { // Determine which node should receive the SetFieldValue. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. @@ -156,7 +157,7 @@ func (t *ClusterCluster) SetFieldValue(index, frame string, columnID uint64, nam if f == nil { return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) } - _, err := f.SetFieldValue(columnID, name, value) + _, err := f.SetValue(columnID, value) if err != nil { return err } From c49fdb7b9bfb96c12cd9754393bfba84f957a8f9 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 1 Jun 2018 22:11:19 -0500 Subject: [PATCH 11/78] rename executor.SetFieldValue() to executor.SetValue() --- executor.go | 14 +++++------ executor_test.go | 62 ++++++++++++++++++++++++------------------------ frame_test.go | 2 +- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/executor.go b/executor.go index c8ea9e544..589b0ace5 100644 --- a/executor.go +++ b/executor.go @@ -138,8 +138,8 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeCount(ctx, index, c, slices, opt) case "SetBit": return e.executeSetBit(ctx, index, c, opt) - case "SetFieldValue": - return nil, e.executeSetFieldValue(ctx, index, c, opt) + case "SetValue": + return nil, e.executeSetValue(ctx, index, c, opt) case "SetRowAttrs": return nil, e.executeSetRowAttrs(ctx, index, c, opt) case "SetColumnAttrs": @@ -1107,14 +1107,14 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C return ret, nil } -// executeSetFieldValue executes a SetFieldValue() call. -func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { +// executeSetValue executes a SetValue() call. +func (e *Executor) executeSetValue(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { // Parse labels. columnID, ok, err := c.UintArg(columnLabel) if err != nil { - return fmt.Errorf("reading SetFieldValue() column: %v", err) + return fmt.Errorf("reading SetValue() column: %v", err) } else if !ok { - return fmt.Errorf("SetFieldValue() column field '%v' required", columnLabel) + return fmt.Errorf("SetValue() column field '%v' required", columnLabel) } // Copy args and remove reserved fields. @@ -1139,7 +1139,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq default: return ErrInvalidFieldValueType } - frame.Stats.Count("SetFieldValue", 1, 1.0) + frame.Stats.Count("SetValue", 1, 1.0) } // Do not forward call if this is already being forwarded. diff --git a/executor_test.go b/executor_test.go index 0b70a9e16..a8cce5194 100644 --- a/executor_test.go +++ b/executor_test.go @@ -263,8 +263,8 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } } -// Ensure a SetFieldValue() query can be executed. -func TestExecutor_Execute_SetFieldValue(t *testing.T) { +// Ensure a SetValue() query can be executed. +func TestExecutor_Execute_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -283,9 +283,9 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { // Set field values. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, f=25)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f=25)`), nil, nil); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, f=10)`), nil, nil); err != nil { + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=100, f=10)`), nil, nil); err != nil { t.Fatal(err) } @@ -321,21 +321,21 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { t.Run("ErrColumnFieldRequired", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnFieldValue", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidFieldValueType", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, f="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { t.Fatalf("unexpected error: %s", err) } }) @@ -591,14 +591,14 @@ func TestExecutor_Execute_MinMax(t *testing.T) { SetBit(frame=x, row=1, col=1) SetBit(frame=x, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(f=20, col=0) - SetFieldValue(f=-5, col=1) - SetFieldValue(f=-5, col=2) - SetFieldValue(f=10, col=3) - SetFieldValue(f=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(f=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(f=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(f=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetValue(f=20, col=0) + SetValue(f=-5, col=1) + SetValue(f=-5, col=2) + SetValue(f=10, col=3) + SetValue(f=30, col=`+strconv.Itoa(SliceWidth)+`) + SetValue(f=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetValue(f=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetValue(f=60, col=`+strconv.Itoa(SliceWidth+1)+`) `), nil, nil); err != nil { t.Fatal(err) } @@ -699,13 +699,13 @@ func TestExecutor_Execute_Sum(t *testing.T) { SetBit(frame=x, row=0, col=0) SetBit(frame=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(foo=20, col=0) - SetFieldValue(bar=2000, col=0) - SetFieldValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(other=1000, col=0) + SetValue(foo=20, col=0) + SetValue(bar=2000, col=0) + SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetValue(foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetValue(foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetValue(other=1000, col=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -820,15 +820,15 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { SetBit(frame=f, row=0, col=0) SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(foo=20, col=50) - SetFieldValue(bar=2000, col=50) - SetFieldValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(other=1000, col=0) - SetFieldValue(edge=100, col=0) - SetFieldValue(edge=-100, col=1) + SetValue(foo=20, col=50) + SetValue(bar=2000, col=50) + SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetValue(foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) + SetValue(foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetValue(other=1000, col=0) + SetValue(edge=100, col=0) + SetValue(edge=-100, col=1) `), nil, nil); err != nil { t.Fatal(err) } diff --git a/frame_test.go b/frame_test.go index 8000886c0..9b7f9a336 100644 --- a/frame_test.go +++ b/frame_test.go @@ -72,7 +72,7 @@ func TestFrame_SetTimeQuantum(t *testing.T) { } // Ensure a frame can set & read a field value. -func TestFrame_SetFieldValue(t *testing.T) { +func TestFrame_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() From 17c8944685412940abeef4758f3b71e0d152e45b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 1 Jun 2018 22:14:45 -0500 Subject: [PATCH 12/78] rename view.SetFieldValue() to view.setValue() --- frame.go | 2 +- view.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frame.go b/frame.go index f460a1224..ff20bddcc 100644 --- a/frame.go +++ b/frame.go @@ -747,7 +747,7 @@ func (f *Frame) SetValue(columnID uint64, value int64) (changed bool, err error) // Determine base value to store. baseValue := uint64(value - field.Min) - return view.SetFieldValue(columnID, field.BitDepth(), baseValue) + return view.setValue(columnID, field.BitDepth(), baseValue) } // FieldSum returns the sum and count for a field. diff --git a/view.go b/view.go index f90bf64e9..a1129c7be 100644 --- a/view.go +++ b/view.go @@ -333,8 +333,8 @@ func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists return frag.FieldValue(columnID, bitDepth) } -// SetFieldValue uses a column of bits to set a multi-bit value. -func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +// setValue uses a column of bits to set a multi-bit value. +func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { From dd6b80f90d604056d51fb97126417b809a203262 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 1 Jun 2018 22:25:26 -0500 Subject: [PATCH 13/78] rename fragment.SetFieldValue() to fragment.SetValue(). Still exported for tests --- fragment.go | 10 +++--- fragment_test.go | 88 ++++++++++++++++++++++++------------------------ test/cluster.go | 25 -------------- utils_test.go | 25 -------------- view.go | 2 +- 5 files changed, 50 insertions(+), 100 deletions(-) diff --git a/fragment.go b/fragment.go index 8044f579b..f2d7ad986 100644 --- a/fragment.go +++ b/fragment.go @@ -510,8 +510,8 @@ func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exi return value, true, nil } -// SetFieldValue uses a column of bits to set a multi-bit value. -func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +// SetValue uses a column of bits to set a multi-bit value. +func (f *Fragment) SetValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() @@ -541,8 +541,8 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) ( return changed, nil } -// importSetFieldValue is a more efficient SetFieldValue just for imports. -func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +// importSetValue is a more efficient SetValue just for imports. +func (f *Fragment) importSetValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { for i := uint(0); i < bitDepth; i++ { if value&(1< Date: Mon, 4 Jun 2018 15:32:43 -0500 Subject: [PATCH 14/78] rename oField to bsiGroup --- api.go | 12 ++--- frame.go | 113 +++++++++++++++++++------------------- frame_internal_test.go | 120 ++++++++++++++++++++--------------------- 3 files changed, 122 insertions(+), 123 deletions(-) diff --git a/api.go b/api.go index c02aedbf2..9a50a3542 100644 --- a/api.go +++ b/api.go @@ -489,7 +489,7 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { } // CreateField creates a new BSI field in the given index and frame. -func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *oField) error { +func (api *API) CreateField(ctx context.Context, indexName string, frameName string, bsig *bsiGroup) error { if err := api.validate(apiCreateField); err != nil { return errors.Wrap(err, "validating api method") } @@ -500,17 +500,17 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str return ErrFrameNotFound } - // Create new field. - if err := f.CreateField(field); err != nil { - return errors.Wrap(err, "creating field") + // Create new bsiGroup. + if err := f.CreateField(bsig); err != nil { + return errors.Wrap(err, "creating bsigroup") } - // Send the create field message to all nodes. + // Send the create bsigroup message to all nodes. err := api.Broadcaster.SendSync( &internal.CreateFieldMessage{ Index: indexName, Frame: frameName, - Field: encodeField(field), + Field: encodeField(bsig), }) if err != nil { api.Logger.Printf("problem sending CreateField message: %s", err) diff --git a/frame.go b/frame.go index ff20bddcc..4fa755e62 100644 --- a/frame.go +++ b/frame.go @@ -64,7 +64,7 @@ type Frame struct { // Frame options. options FrameOptions - fields []*oField + fields []*bsiGroup Logger Logger } @@ -325,19 +325,19 @@ func (f *Frame) applyOptions(opt FrameOptions) error { f.options.Max = opt.Max f.options.TimeQuantum = "" - // Create new field. - field := &oField{ + // Create new bsiGroup. + bsig := &bsiGroup{ Name: f.name, Type: FieldTypeInt, Min: opt.Min, Max: opt.Max, } - // Validate field. - if err := ValidateField(field); err != nil { + // Validate bsiGroup. + if err := ValidateField(bsig); err != nil { return err } - if err := f.CreateField(field); err != nil { - return errors.Wrap(err, "creating field") + if err := f.CreateField(bsig); err != nil { + return errors.Wrap(err, "creating bsigroup") } case FrameTypeTime: f.options.Type = opt.Type @@ -379,7 +379,7 @@ func (f *Frame) Close() error { } // Field returns a field by name. -func (f *Frame) Field(name string) *oField { +func (f *Frame) Field(name string) *bsiGroup { f.mu.RLock() defer f.mu.RUnlock() for _, field := range f.fields { @@ -401,12 +401,12 @@ func (f *Frame) hasField(name string) bool { } // CreateField creates a new field on the frame. -func (f *Frame) CreateField(field *oField) error { +func (f *Frame) CreateField(bsig *bsiGroup) error { f.mu.Lock() defer f.mu.Unlock() - // Append field. - if err := f.addField(field); err != nil { + // Append bsiGroup. + if err := f.addField(bsig); err != nil { return err } f.saveMeta() @@ -414,15 +414,15 @@ func (f *Frame) CreateField(field *oField) error { } // addField adds a single field to fields. -func (f *Frame) addField(field *oField) error { - if err := ValidateField(field); err != nil { - return errors.Wrap(err, "validating field") - } else if f.hasField(field.Name) { +func (f *Frame) addField(bsig *bsiGroup) error { + if err := ValidateField(bsig); err != nil { + return errors.Wrap(err, "validating bsigroup") + } else if f.hasField(bsig.Name) { return ErrFieldExists } - // Add field to list. - f.fields = append(f.fields, field) + // Add bsiGroup to list. + f.fields = append(f.fields, bsig) // Sort fields by name. sort.Slice(f.fields, func(i, j int) bool { @@ -1084,9 +1084,8 @@ func IsValidFieldType(v string) bool { } } -// TODO: finish unexporting this. also, rename it. -// oField represents a range field on a frame. -type oField struct { +// bsiGroup represents a range field on a frame. +type bsiGroup struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` Min int64 `json:"min,omitempty"` @@ -1094,9 +1093,9 @@ type oField struct { } // BitDepth returns the number of bits required to store a value between min & max. -func (f *oField) BitDepth() uint { +func (b *bsiGroup) BitDepth() uint { for i := uint(0); i < 63; i++ { - if f.Max-f.Min < (1 << i) { + if b.Max-b.Min < (1 << i) { return i } } @@ -1115,80 +1114,80 @@ func (f *oField) BitDepth() uint { // In order to make this work, we effectively need to change the operator to LTE. // Executor.executeFieldRangeSlice() takes this into account and returns // `frag.FieldNotNull(field.BitDepth())` in such instances. -func (f *oField) BaseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { +func (b *bsiGroup) BaseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { if op == pql.GT || op == pql.GTE { - if value > f.Max { + if value > b.Max { return baseValue, true - } else if value > f.Min { - baseValue = uint64(value - f.Min) + } else if value > b.Min { + baseValue = uint64(value - b.Min) } } else if op == pql.LT || op == pql.LTE { - if value < f.Min { + if value < b.Min { return baseValue, true - } else if value > f.Max { - baseValue = uint64(f.Max - f.Min) + } else if value > b.Max { + baseValue = uint64(b.Max - b.Min) } else { - baseValue = uint64(value - f.Min) + baseValue = uint64(value - b.Min) } } else if op == pql.EQ || op == pql.NEQ { - if value < f.Min || value > f.Max { + if value < b.Min || value > b.Max { return baseValue, true } - baseValue = uint64(value - f.Min) + baseValue = uint64(value - b.Min) } return baseValue, false } // BaseValueBetween adjusts the min/max value to align with the range for Field. -func (f *oField) BaseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { - if max < f.Min || min > f.Max { +func (b *bsiGroup) BaseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { + if max < b.Min || min > b.Max { return baseValueMin, baseValueMax, true } // Adjust min/max to range. - if min > f.Min { - baseValueMin = uint64(min - f.Min) + if min > b.Min { + baseValueMin = uint64(min - b.Min) } // Make sure the high value of the BETWEEN does not exceed BitDepth. - if max > f.Max { - baseValueMax = uint64(f.Max - f.Min) - } else if max > f.Min { - baseValueMax = uint64(max - f.Min) + if max > b.Max { + baseValueMax = uint64(b.Max - b.Min) + } else if max > b.Min { + baseValueMax = uint64(max - b.Min) } return baseValueMin, baseValueMax, false } -func ValidateField(f *oField) error { - if f.Name == "" { +func ValidateField(b *bsiGroup) error { + if b.Name == "" { return ErrFieldNameRequired - } else if !IsValidFieldType(f.Type) { + } else if !IsValidFieldType(b.Type) { return ErrInvalidFieldType - } else if f.Min > f.Max { + } else if b.Min > b.Max { return ErrInvalidFieldRange } return nil } -func encodeField(f *oField) *internal.Field { - if f == nil { +func encodeField(b *bsiGroup) *internal.Field { + if b == nil { return nil } return &internal.Field{ - Name: f.Name, - Type: f.Type, - Min: int64(f.Min), - Max: int64(f.Max), + Name: b.Name, + Type: b.Type, + Min: int64(b.Min), + Max: int64(b.Max), } } -func decodeField(f *internal.Field) *oField { - if f == nil { +func decodeField(b *internal.Field) *bsiGroup { + if b == nil { return nil } - return &oField{ - Name: f.Name, - Type: f.Type, - Min: f.Min, - Max: f.Max, + return &bsiGroup{ + Name: b.Name, + Type: b.Type, + Min: b.Min, + Max: b.Max, } } diff --git a/frame_internal_test.go b/frame_internal_test.go index f51ad95e9..c52b76658 100644 --- a/frame_internal_test.go +++ b/frame_internal_test.go @@ -23,21 +23,21 @@ import ( // Ensure a field can adjust to its baseValue. func TestField_BaseValue(t *testing.T) { - f0 := &oField{ - Name: "f0", + b0 := &bsiGroup{ + Name: "b0", Type: FieldTypeInt, Min: -100, Max: 900, } - f1 := &oField{ - Name: "f1", + b1 := &bsiGroup{ + Name: "b1", Type: FieldTypeInt, Min: 0, Max: 1000, } - f2 := &oField{ - Name: "f2", + b2 := &bsiGroup{ + Name: "b2", Type: FieldTypeInt, Min: 100, Max: 1100, @@ -46,60 +46,60 @@ func TestField_BaseValue(t *testing.T) { t.Run("Normal Condition", func(t *testing.T) { for _, tt := range []struct { - f *oField + f *bsiGroup op pql.Token val int64 expBaseValue uint64 expOutOfRange bool }{ // LT - {f0, pql.LT, 5, 105, false}, - {f0, pql.LT, -8, 92, false}, - {f0, pql.LT, -108, 0, true}, - {f0, pql.LT, 1005, 1000, false}, - {f0, pql.LT, 0, 100, false}, + {b0, pql.LT, 5, 105, false}, + {b0, pql.LT, -8, 92, false}, + {b0, pql.LT, -108, 0, true}, + {b0, pql.LT, 1005, 1000, false}, + {b0, pql.LT, 0, 100, false}, - {f1, pql.LT, 5, 5, false}, - {f1, pql.LT, -8, 0, true}, - {f1, pql.LT, 1005, 1000, false}, - {f1, pql.LT, 0, 0, false}, + {b1, pql.LT, 5, 5, false}, + {b1, pql.LT, -8, 0, true}, + {b1, pql.LT, 1005, 1000, false}, + {b1, pql.LT, 0, 0, false}, - {f2, pql.LT, 5, 0, true}, - {f2, pql.LT, -8, 0, true}, - {f2, pql.LT, 105, 5, false}, - {f2, pql.LT, 1105, 1000, false}, + {b2, pql.LT, 5, 0, true}, + {b2, pql.LT, -8, 0, true}, + {b2, pql.LT, 105, 5, false}, + {b2, pql.LT, 1105, 1000, false}, // GT - {f0, pql.GT, -105, 0, false}, - {f0, pql.GT, 5, 105, false}, - {f0, pql.GT, 905, 0, true}, - {f0, pql.GT, 0, 100, false}, + {b0, pql.GT, -105, 0, false}, + {b0, pql.GT, 5, 105, false}, + {b0, pql.GT, 905, 0, true}, + {b0, pql.GT, 0, 100, false}, - {f1, pql.GT, 5, 5, false}, - {f1, pql.GT, -8, 0, false}, - {f1, pql.GT, 1005, 0, true}, - {f1, pql.GT, 0, 0, false}, + {b1, pql.GT, 5, 5, false}, + {b1, pql.GT, -8, 0, false}, + {b1, pql.GT, 1005, 0, true}, + {b1, pql.GT, 0, 0, false}, - {f2, pql.GT, 5, 0, false}, - {f2, pql.GT, -8, 0, false}, - {f2, pql.GT, 105, 5, false}, - {f2, pql.GT, 1105, 0, true}, + {b2, pql.GT, 5, 0, false}, + {b2, pql.GT, -8, 0, false}, + {b2, pql.GT, 105, 5, false}, + {b2, pql.GT, 1105, 0, true}, // EQ - {f0, pql.EQ, -105, 0, true}, - {f0, pql.EQ, 5, 105, false}, - {f0, pql.EQ, 905, 0, true}, - {f0, pql.EQ, 0, 100, false}, + {b0, pql.EQ, -105, 0, true}, + {b0, pql.EQ, 5, 105, false}, + {b0, pql.EQ, 905, 0, true}, + {b0, pql.EQ, 0, 100, false}, - {f1, pql.EQ, 5, 5, false}, - {f1, pql.EQ, -8, 0, true}, - {f1, pql.EQ, 1005, 0, true}, - {f1, pql.EQ, 0, 0, false}, + {b1, pql.EQ, 5, 5, false}, + {b1, pql.EQ, -8, 0, true}, + {b1, pql.EQ, 1005, 0, true}, + {b1, pql.EQ, 0, 0, false}, - {f2, pql.EQ, 5, 0, true}, - {f2, pql.EQ, -8, 0, true}, - {f2, pql.EQ, 105, 5, false}, - {f2, pql.EQ, 1105, 0, true}, + {b2, pql.EQ, 5, 0, true}, + {b2, pql.EQ, -8, 0, true}, + {b2, pql.EQ, 105, 5, false}, + {b2, pql.EQ, 1105, 0, true}, } { bv, oor := tt.f.BaseValue(tt.op, tt.val) if oor != tt.expOutOfRange { @@ -112,7 +112,7 @@ func TestField_BaseValue(t *testing.T) { t.Run("Betwween Condition", func(t *testing.T) { for _, tt := range []struct { - f *oField + f *bsiGroup predMin int64 predMax int64 expBaseValueMin uint64 @@ -120,23 +120,23 @@ func TestField_BaseValue(t *testing.T) { expOutOfRange bool }{ - {f0, -205, -105, 0, 0, true}, - {f0, -105, 80, 0, 180, false}, - {f0, 5, 20, 105, 120, false}, - {f0, 20, 1005, 120, 1000, false}, - {f0, 1005, 2000, 0, 0, true}, + {b0, -205, -105, 0, 0, true}, + {b0, -105, 80, 0, 180, false}, + {b0, 5, 20, 105, 120, false}, + {b0, 20, 1005, 120, 1000, false}, + {b0, 1005, 2000, 0, 0, true}, - {f1, -105, -5, 0, 0, true}, - {f1, -5, 20, 0, 20, false}, - {f1, 5, 20, 5, 20, false}, - {f1, 20, 1005, 20, 1000, false}, - {f1, 1005, 2000, 0, 0, true}, + {b1, -105, -5, 0, 0, true}, + {b1, -5, 20, 0, 20, false}, + {b1, 5, 20, 5, 20, false}, + {b1, 20, 1005, 20, 1000, false}, + {b1, 1005, 2000, 0, 0, true}, - {f2, 5, 95, 0, 0, true}, - {f2, 95, 120, 0, 20, false}, - {f2, 105, 120, 5, 20, false}, - {f2, 120, 1105, 20, 1000, false}, - {f2, 1105, 2000, 0, 0, true}, + {b2, 5, 95, 0, 0, true}, + {b2, 95, 120, 0, 20, false}, + {b2, 105, 120, 5, 20, false}, + {b2, 120, 1105, 20, 1000, false}, + {b2, 1105, 2000, 0, 0, true}, } { min, max, oor := tt.f.BaseValueBetween(tt.predMin, tt.predMax) if oor != tt.expOutOfRange { From 63e41b912b253313dd57af75140520a21c6209d2 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Jun 2018 15:45:52 -0500 Subject: [PATCH 15/78] rename Frame.fields to Frame.bsiGroups --- frame.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/frame.go b/frame.go index 4fa755e62..6ad64a352 100644 --- a/frame.go +++ b/frame.go @@ -64,7 +64,7 @@ type Frame struct { // Frame options. options FrameOptions - fields []*bsiGroup + bsiGroups []*bsiGroup Logger Logger } @@ -382,9 +382,9 @@ func (f *Frame) Close() error { func (f *Frame) Field(name string) *bsiGroup { f.mu.RLock() defer f.mu.RUnlock() - for _, field := range f.fields { - if field.Name == name { - return field + for _, bsig := range f.bsiGroups { + if bsig.Name == name { + return bsig } } return nil @@ -392,8 +392,8 @@ func (f *Frame) Field(name string) *bsiGroup { // hasField returns true if a field exists on the frame. func (f *Frame) hasField(name string) bool { - for _, fld := range f.fields { - if fld.Name == name { + for _, bsig := range f.bsiGroups { + if bsig.Name == name { return true } } @@ -422,11 +422,11 @@ func (f *Frame) addField(bsig *bsiGroup) error { } // Add bsiGroup to list. - f.fields = append(f.fields, bsig) + f.bsiGroups = append(f.bsiGroups, bsig) // Sort fields by name. - sort.Slice(f.fields, func(i, j int) bool { - return f.fields[i].Name < f.fields[j].Name + sort.Slice(f.bsiGroups, func(i, j int) bool { + return f.bsiGroups[i].Name < f.bsiGroups[j].Name }) return nil @@ -459,10 +459,10 @@ func (f *Frame) DeleteField(name string) error { // deleteField removes a single field from fields. func (f *Frame) deleteField(name string) error { - for i, field := range f.fields { - if field.Name == name { - copy(f.fields[i:], f.fields[i+1:]) - f.fields, f.fields[len(f.fields)-1] = f.fields[:len(f.fields)-1], nil + for i, bsig := range f.bsiGroups { + if bsig.Name == name { + copy(f.bsiGroups[i:], f.bsiGroups[i+1:]) + f.bsiGroups, f.bsiGroups[len(f.bsiGroups)-1] = f.bsiGroups[:len(f.bsiGroups)-1], nil return nil } } From 5d39ba25dfac9731d79fc02c00f329ce8461ded1 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Jun 2018 15:49:02 -0500 Subject: [PATCH 16/78] rename FieldTypeInt to bsiGroupTypeInt --- frame.go | 8 ++++---- frame_internal_test.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/frame.go b/frame.go index 6ad64a352..26d1bdf3f 100644 --- a/frame.go +++ b/frame.go @@ -328,7 +328,7 @@ func (f *Frame) applyOptions(opt FrameOptions) error { // Create new bsiGroup. bsig := &bsiGroup{ Name: f.name, - Type: FieldTypeInt, + Type: bsiGroupTypeInt, Min: opt.Min, Max: opt.Max, } @@ -1070,14 +1070,14 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { } } -// List of field data types. +// List of bsiGroup types. const ( - FieldTypeInt = "int" + bsiGroupTypeInt = "int" ) func IsValidFieldType(v string) bool { switch v { - case FieldTypeInt: + case bsiGroupTypeInt: return true default: return false diff --git a/frame_internal_test.go b/frame_internal_test.go index c52b76658..749e4e720 100644 --- a/frame_internal_test.go +++ b/frame_internal_test.go @@ -25,20 +25,20 @@ import ( func TestField_BaseValue(t *testing.T) { b0 := &bsiGroup{ Name: "b0", - Type: FieldTypeInt, + Type: bsiGroupTypeInt, Min: -100, Max: 900, } b1 := &bsiGroup{ Name: "b1", - Type: FieldTypeInt, + Type: bsiGroupTypeInt, Min: 0, Max: 1000, } b2 := &bsiGroup{ Name: "b2", - Type: FieldTypeInt, + Type: bsiGroupTypeInt, Min: 100, Max: 1100, } From 54d94f1acaf6ff60e5db769027ecd24a0fc188b6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Jun 2018 16:03:42 -0500 Subject: [PATCH 17/78] rename CreateField to createBSIGroup --- api.go | 2 +- frame.go | 12 ++++++------ server.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api.go b/api.go index 9a50a3542..0dc153380 100644 --- a/api.go +++ b/api.go @@ -501,7 +501,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str } // Create new bsiGroup. - if err := f.CreateField(bsig); err != nil { + if err := f.createBSIGroup(bsig); err != nil { return errors.Wrap(err, "creating bsigroup") } diff --git a/frame.go b/frame.go index 26d1bdf3f..fc52bf3d2 100644 --- a/frame.go +++ b/frame.go @@ -333,10 +333,10 @@ func (f *Frame) applyOptions(opt FrameOptions) error { Max: opt.Max, } // Validate bsiGroup. - if err := ValidateField(bsig); err != nil { + if err := bsig.validate(); err != nil { return err } - if err := f.CreateField(bsig); err != nil { + if err := f.createBSIGroup(bsig); err != nil { return errors.Wrap(err, "creating bsigroup") } case FrameTypeTime: @@ -400,8 +400,8 @@ func (f *Frame) hasField(name string) bool { return false } -// CreateField creates a new field on the frame. -func (f *Frame) CreateField(bsig *bsiGroup) error { +// createBSIGroup creates a new field on the frame. +func (f *Frame) createBSIGroup(bsig *bsiGroup) error { f.mu.Lock() defer f.mu.Unlock() @@ -415,7 +415,7 @@ func (f *Frame) CreateField(bsig *bsiGroup) error { // addField adds a single field to fields. func (f *Frame) addField(bsig *bsiGroup) error { - if err := ValidateField(bsig); err != nil { + if err := bsig.validate(); err != nil { return errors.Wrap(err, "validating bsigroup") } else if f.hasField(bsig.Name) { return ErrFieldExists @@ -1156,7 +1156,7 @@ func (b *bsiGroup) BaseValueBetween(min, max int64) (baseValueMin, baseValueMax return baseValueMin, baseValueMax, false } -func ValidateField(b *bsiGroup) error { +func (b *bsiGroup) validate() error { if b.Name == "" { return ErrFieldNameRequired } else if !IsValidFieldType(b.Type) { diff --git a/server.go b/server.go index 320fd174c..e7c566cbf 100644 --- a/server.go +++ b/server.go @@ -473,7 +473,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { case *internal.CreateFieldMessage: f := s.Holder.Frame(obj.Index, obj.Frame) field := decodeField(obj.Field) - if err := f.CreateField(field); err != nil { + if err := f.createBSIGroup(field); err != nil { return err } case *internal.DeleteFieldMessage: From b10463485e03b04e3c0544940476cbd9ca179e9b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Jun 2018 16:06:41 -0500 Subject: [PATCH 18/78] rename Frame.Field() to Frame.bsiGroup() --- executor.go | 12 ++++++------ frame.go | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/executor.go b/executor.go index 589b0ace5..25d902591 100644 --- a/executor.go +++ b/executor.go @@ -381,7 +381,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq return ValCount{}, nil } - field := frame.Field(fieldName) + field := frame.bsiGroup(fieldName) if field == nil { return ValCount{}, nil } @@ -420,7 +420,7 @@ func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pq return ValCount{}, nil } - field := frame.Field(fieldName) + field := frame.bsiGroup(fieldName) if field == nil { return ValCount{}, nil } @@ -459,7 +459,7 @@ func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pq return ValCount{}, nil } - field := frame.Field(fieldName) + field := frame.bsiGroup(fieldName) if field == nil { return ValCount{}, nil } @@ -800,7 +800,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * // Handle `!= null`. if cond.Op == pql.NEQ && cond.Value == nil { // Find field. - field := f.Field(fieldName) + field := f.bsiGroup(fieldName) if field == nil { return nil, ErrFieldNotFound } @@ -830,7 +830,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * // here is because we need the call to be slice-specific. // Find field. - field := f.Field(fieldName) + field := f.bsiGroup(fieldName) if field == nil { return nil, ErrFieldNotFound } @@ -863,7 +863,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * } // Find field. - field := f.Field(fieldName) + field := f.bsiGroup(fieldName) if field == nil { return nil, ErrFieldNotFound } diff --git a/frame.go b/frame.go index fc52bf3d2..b2d893792 100644 --- a/frame.go +++ b/frame.go @@ -378,8 +378,8 @@ func (f *Frame) Close() error { return nil } -// Field returns a field by name. -func (f *Frame) Field(name string) *bsiGroup { +// bsiGroup returns a field by name. +func (f *Frame) bsiGroup(name string) *bsiGroup { f.mu.RLock() defer f.mu.RUnlock() for _, bsig := range f.bsiGroups { @@ -706,7 +706,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change // FieldValue reads a field value for a column. func (f *Frame) FieldValue(columnID uint64, name string) (value int64, exists bool, err error) { - field := f.Field(name) + field := f.bsiGroup(name) if field == nil { return 0, false, ErrFieldNotFound } @@ -729,7 +729,7 @@ func (f *Frame) FieldValue(columnID uint64, name string) (value int64, exists bo // SetValue sets a field value for a column. func (f *Frame) SetValue(columnID uint64, value int64) (changed bool, err error) { // Fetch field and validate value. - field := f.Field(f.name) + field := f.bsiGroup(f.name) if field == nil { return false, ErrFieldNotFound } else if value < field.Min { @@ -753,7 +753,7 @@ func (f *Frame) SetValue(columnID uint64, value int64) (changed bool, err error) // FieldSum returns the sum and count for a field. // An optional filtering row can be provided. func (f *Frame) FieldSum(filter *Row, name string) (sum, count int64, err error) { - field := f.Field(name) + field := f.bsiGroup(name) if field == nil { return 0, 0, ErrFieldNotFound } @@ -773,7 +773,7 @@ func (f *Frame) FieldSum(filter *Row, name string) (sum, count int64, err error) // FieldMin returns the min for a field. // An optional filtering row can be provided. func (f *Frame) FieldMin(filter *Row, name string) (min, count int64, err error) { - field := f.Field(name) + field := f.bsiGroup(name) if field == nil { return 0, 0, ErrFieldNotFound } @@ -793,7 +793,7 @@ func (f *Frame) FieldMin(filter *Row, name string) (min, count int64, err error) // FieldMax returns the max for a field. // An optional filtering row can be provided. func (f *Frame) FieldMax(filter *Row, name string) (max, count int64, err error) { - field := f.Field(name) + field := f.bsiGroup(name) if field == nil { return 0, 0, ErrFieldNotFound } @@ -812,7 +812,7 @@ func (f *Frame) FieldMax(filter *Row, name string) (max, count int64, err error) func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Row, error) { // Retrieve and validate field. - field := f.Field(name) + field := f.bsiGroup(name) if field == nil { return nil, ErrFieldNotFound } else if predicate < field.Min || predicate > field.Max { @@ -835,7 +835,7 @@ func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Row, er func (f *Frame) FieldRangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { // Retrieve and validate field. - field := f.Field(name) + field := f.bsiGroup(name) if field == nil { return nil, ErrFieldNotFound } else if predicateMin > predicateMax { @@ -917,7 +917,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { viewName := ViewFieldPrefix + fieldName // Get the field so we know bitDepth. - field := f.Field(fieldName) + field := f.bsiGroup(fieldName) if field == nil { return fmt.Errorf("Field does not exist: %s", fieldName) } From dcf4daf24e5285296571b5a21d09347345a2367d Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Jun 2018 16:47:08 -0500 Subject: [PATCH 19/78] rename a lot of *Field cases to *BSIGroup --- executor.go | 20 +++++------ executor_test.go | 2 +- frame.go | 89 ++++++++++++++++++++++++------------------------ frame_test.go | 6 ++-- pilosa.go | 20 +++++------ view.go | 4 +-- 6 files changed, 71 insertions(+), 70 deletions(-) diff --git a/executor.go b/executor.go index 25d902591..43b44c315 100644 --- a/executor.go +++ b/executor.go @@ -386,7 +386,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+fieldName, slice) if fragment == nil { return ValCount{}, nil } @@ -425,7 +425,7 @@ func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pq return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+fieldName, slice) if fragment == nil { return ValCount{}, nil } @@ -464,7 +464,7 @@ func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pq return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+fieldName, slice) if fragment == nil { return ValCount{}, nil } @@ -802,11 +802,11 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * // Find field. field := f.bsiGroup(fieldName) if field == nil { - return nil, ErrFieldNotFound + return nil, ErrBSIGroupNotFound } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, frame, viewBSIGroupPrefix+fieldName, slice) if frag == nil { return NewRow(), nil } @@ -832,7 +832,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * // Find field. field := f.bsiGroup(fieldName) if field == nil { - return nil, ErrFieldNotFound + return nil, ErrBSIGroupNotFound } baseValueMin, baseValueMax, outOfRange := field.BaseValueBetween(predicates[0], predicates[1]) @@ -841,7 +841,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, frame, viewBSIGroupPrefix+fieldName, slice) if frag == nil { return NewRow(), nil } @@ -865,7 +865,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * // Find field. field := f.bsiGroup(fieldName) if field == nil { - return nil, ErrFieldNotFound + return nil, ErrBSIGroupNotFound } baseValue, outOfRange := field.BaseValue(cond.Op, value) @@ -874,7 +874,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, frame, viewBSIGroupPrefix+fieldName, slice) if frag == nil { return NewRow(), nil } @@ -1137,7 +1137,7 @@ func (e *Executor) executeSetValue(ctx context.Context, index string, c *pql.Cal return err } default: - return ErrInvalidFieldValueType + return ErrInvalidBSIGroupValueType } frame.Stats.Count("SetValue", 1, 1.0) } diff --git a/executor_test.go b/executor_test.go index a8cce5194..3429529ee 100644 --- a/executor_test.go +++ b/executor_test.go @@ -951,7 +951,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("ErrFieldNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, bad_field >= 20)`), nil, nil); err != pilosa.ErrBSIGroupNotFound { t.Fatal(err) } }) diff --git a/frame.go b/frame.go index b2d893792..6e2ef0bf7 100644 --- a/frame.go +++ b/frame.go @@ -378,7 +378,7 @@ func (f *Frame) Close() error { return nil } -// bsiGroup returns a field by name. +// bsiGroup returns a bsiGroup by name. func (f *Frame) bsiGroup(name string) *bsiGroup { f.mu.RLock() defer f.mu.RUnlock() @@ -390,8 +390,8 @@ func (f *Frame) bsiGroup(name string) *bsiGroup { return nil } -// hasField returns true if a field exists on the frame. -func (f *Frame) hasField(name string) bool { +// hasBSIGroup returns true if a bsiGroup exists on the frame. +func (f *Frame) hasBSIGroup(name string) bool { for _, bsig := range f.bsiGroups { if bsig.Name == name { return true @@ -400,25 +400,25 @@ func (f *Frame) hasField(name string) bool { return false } -// createBSIGroup creates a new field on the frame. +// createBSIGroup creates a new bsiGroup on the frame. func (f *Frame) createBSIGroup(bsig *bsiGroup) error { f.mu.Lock() defer f.mu.Unlock() // Append bsiGroup. - if err := f.addField(bsig); err != nil { + if err := f.addBSIGroup(bsig); err != nil { return err } f.saveMeta() return nil } -// addField adds a single field to fields. -func (f *Frame) addField(bsig *bsiGroup) error { +// addBSIGroup adds a single bsiGroup to bsiGroups. +func (f *Frame) addBSIGroup(bsig *bsiGroup) error { if err := bsig.validate(); err != nil { return errors.Wrap(err, "validating bsigroup") - } else if f.hasField(bsig.Name) { - return ErrFieldExists + } else if f.hasBSIGroup(bsig.Name) { + return ErrBSIGroupExists } // Add bsiGroup to list. @@ -432,18 +432,19 @@ func (f *Frame) addField(bsig *bsiGroup) error { return nil } +// TODO: merge this into the un-exported deleteBSIGroup. // DeleteField deletes an existing field on the schema. func (f *Frame) DeleteField(name string) error { f.mu.Lock() defer f.mu.Unlock() // Remove field. - if err := f.deleteField(name); err != nil { + if err := f.deleteBSIGroup(name); err != nil { return err } // Remove views. - viewName := ViewFieldPrefix + name + viewName := viewBSIGroupPrefix + name if view := f.views[viewName]; view != nil { delete(f.views, viewName) @@ -457,8 +458,8 @@ func (f *Frame) DeleteField(name string) error { return nil } -// deleteField removes a single field from fields. -func (f *Frame) deleteField(name string) error { +// deleteBSIGroup removes a single bsiGroup from bsiGroups. +func (f *Frame) deleteBSIGroup(name string) error { for i, bsig := range f.bsiGroups { if bsig.Name == name { copy(f.bsiGroups[i:], f.bsiGroups[i+1:]) @@ -466,7 +467,7 @@ func (f *Frame) deleteField(name string) error { return nil } } - return ErrFieldNotFound + return ErrBSIGroupNotFound } // TimeQuantum returns the time quantum for the frame. @@ -708,11 +709,11 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change func (f *Frame) FieldValue(columnID uint64, name string) (value int64, exists bool, err error) { field := f.bsiGroup(name) if field == nil { - return 0, false, ErrFieldNotFound + return 0, false, ErrBSIGroupNotFound } // Fetch target view. - view := f.View(ViewFieldPrefix + name) + view := f.View(viewBSIGroupPrefix + name) if view == nil { return 0, false, nil } @@ -731,15 +732,15 @@ func (f *Frame) SetValue(columnID uint64, value int64) (changed bool, err error) // Fetch field and validate value. field := f.bsiGroup(f.name) if field == nil { - return false, ErrFieldNotFound + return false, ErrBSIGroupNotFound } else if value < field.Min { - return false, ErrFieldValueTooLow + return false, ErrBSIGroupValueTooLow } else if value > field.Max { - return false, ErrFieldValueTooHigh + return false, ErrBSIGroupValueTooHigh } // Fetch target view. - view, err := f.CreateViewIfNotExists(ViewFieldPrefix + f.name) + view, err := f.CreateViewIfNotExists(viewBSIGroupPrefix + f.name) if err != nil { return false, errors.Wrap(err, "creating view") } @@ -755,10 +756,10 @@ func (f *Frame) SetValue(columnID uint64, value int64) (changed bool, err error) func (f *Frame) FieldSum(filter *Row, name string) (sum, count int64, err error) { field := f.bsiGroup(name) if field == nil { - return 0, 0, ErrFieldNotFound + return 0, 0, ErrBSIGroupNotFound } - view := f.View(ViewFieldPrefix + name) + view := f.View(viewBSIGroupPrefix + name) if view == nil { return 0, 0, nil } @@ -770,24 +771,24 @@ func (f *Frame) FieldSum(filter *Row, name string) (sum, count int64, err error) return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil } -// FieldMin returns the min for a field. +// FieldMin returns the min for a bsiGroup. // An optional filtering row can be provided. func (f *Frame) FieldMin(filter *Row, name string) (min, count int64, err error) { - field := f.bsiGroup(name) - if field == nil { - return 0, 0, ErrFieldNotFound + bsig := f.bsiGroup(name) + if bsig == nil { + return 0, 0, ErrBSIGroupNotFound } - view := f.View(ViewFieldPrefix + name) + view := f.View(viewBSIGroupPrefix + name) if view == nil { return 0, 0, nil } - vmin, vcount, err := view.FieldMin(filter, field.BitDepth()) + vmin, vcount, err := view.FieldMin(filter, bsig.BitDepth()) if err != nil { return 0, 0, err } - return int64(vmin) + field.Min, int64(vcount), nil + return int64(vmin) + bsig.Min, int64(vcount), nil } // FieldMax returns the max for a field. @@ -795,10 +796,10 @@ func (f *Frame) FieldMin(filter *Row, name string) (min, count int64, err error) func (f *Frame) FieldMax(filter *Row, name string) (max, count int64, err error) { field := f.bsiGroup(name) if field == nil { - return 0, 0, ErrFieldNotFound + return 0, 0, ErrBSIGroupNotFound } - view := f.View(ViewFieldPrefix + name) + view := f.View(viewBSIGroupPrefix + name) if view == nil { return 0, 0, nil } @@ -814,13 +815,13 @@ func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Row, er // Retrieve and validate field. field := f.bsiGroup(name) if field == nil { - return nil, ErrFieldNotFound + return nil, ErrBSIGroupNotFound } else if predicate < field.Min || predicate > field.Max { return nil, nil } // Retrieve field's view. - view := f.View(ViewFieldPrefix + name) + view := f.View(viewBSIGroupPrefix + name) if view == nil { return nil, nil } @@ -837,13 +838,13 @@ func (f *Frame) FieldRangeBetween(name string, predicateMin, predicateMax int64) // Retrieve and validate field. field := f.bsiGroup(name) if field == nil { - return nil, ErrFieldNotFound + return nil, ErrBSIGroupNotFound } else if predicateMin > predicateMax { return nil, ErrInvalidBetweenValue } // Retrieve field's view. - view := f.View(ViewFieldPrefix + name) + view := f.View(viewBSIGroupPrefix + name) if view == nil { return nil, nil } @@ -915,7 +916,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // ImportValue bulk imports range-encoded value data. func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { - viewName := ViewFieldPrefix + fieldName + viewName := viewBSIGroupPrefix + fieldName // Get the field so we know bitDepth. field := f.bsiGroup(fieldName) if field == nil { @@ -927,9 +928,9 @@ func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64 for i := range columnIDs { columnID, value := columnIDs[i], values[i] if int64(value) > field.Max { - return fmt.Errorf("%v, columnID=%v, value=%v", ErrFieldValueTooHigh, columnID, value) + return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooHigh, columnID, value) } else if int64(value) < field.Min { - return fmt.Errorf("%v, columnID=%v, value=%v", ErrFieldValueTooLow, columnID, value) + return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooLow, columnID, value) } // Attach value to each field view. @@ -1025,7 +1026,7 @@ func (o *FrameOptions) Validate() error { // TODO: cacheType, cacheSize validation case FrameTypeInt: if o.Min > o.Max { - return ErrInvalidFieldRange + return ErrInvalidBSIGroupRange } case FrameTypeTime: if o.TimeQuantum == "" || !o.TimeQuantum.Valid() { @@ -1075,7 +1076,7 @@ const ( bsiGroupTypeInt = "int" ) -func IsValidFieldType(v string) bool { +func isValidBSIGroupType(v string) bool { switch v { case bsiGroupTypeInt: return true @@ -1158,11 +1159,11 @@ func (b *bsiGroup) BaseValueBetween(min, max int64) (baseValueMin, baseValueMax func (b *bsiGroup) validate() error { if b.Name == "" { - return ErrFieldNameRequired - } else if !IsValidFieldType(b.Type) { - return ErrInvalidFieldType + return ErrBSIGroupNameRequired + } else if !isValidBSIGroupType(b.Type) { + return ErrInvalidBSIGroupType } else if b.Min > b.Max { - return ErrInvalidFieldRange + return ErrInvalidBSIGroupRange } return nil } diff --git a/frame_test.go b/frame_test.go index 9b7f9a336..6cc75f6be 100644 --- a/frame_test.go +++ b/frame_test.go @@ -159,7 +159,7 @@ func TestFrame_SetValue(t *testing.T) { } // Set value. - if _, err := f.SetValue(100, 21); err != pilosa.ErrFieldNotFound { + if _, err := f.SetValue(100, 21); err != pilosa.ErrBSIGroupNotFound { t.Fatalf("unexpected error: %s", err) } }) @@ -178,7 +178,7 @@ func TestFrame_SetValue(t *testing.T) { } // Set value. - if _, err := f.SetValue(100, 15); err != pilosa.ErrFieldValueTooLow { + if _, err := f.SetValue(100, 15); err != pilosa.ErrBSIGroupValueTooLow { t.Fatalf("unexpected error: %s", err) } }) @@ -197,7 +197,7 @@ func TestFrame_SetValue(t *testing.T) { } // Set value. - if _, err := f.SetValue(100, 31); err != pilosa.ErrFieldValueTooHigh { + if _, err := f.SetValue(100, 31); err != pilosa.ErrBSIGroupValueTooHigh { t.Fatalf("unexpected error: %s", err) } }) diff --git a/pilosa.go b/pilosa.go index 7ddea28eb..21bfdd822 100644 --- a/pilosa.go +++ b/pilosa.go @@ -36,16 +36,16 @@ var ( ErrFrameExists = errors.New("frame already exists") ErrFrameNotFound = errors.New("frame not found") - ErrFieldNotFound = errors.New("field not found") - ErrFieldExists = errors.New("field already exists") - ErrFieldNameRequired = errors.New("field name required") - ErrInvalidFieldType = errors.New("invalid field type") - ErrInvalidFieldRange = errors.New("invalid field range") - ErrInvalidFieldValueType = errors.New("invalid field value type") - ErrFieldValueTooLow = errors.New("field value too low") - ErrFieldValueTooHigh = errors.New("field value too high") - ErrInvalidRangeOperation = errors.New("invalid range operation") - ErrInvalidBetweenValue = errors.New("invalid value for between operation") + ErrBSIGroupNotFound = errors.New("bsigroup not found") + ErrBSIGroupExists = errors.New("bsigroup already exists") + ErrBSIGroupNameRequired = errors.New("bsigroup name required") + ErrInvalidBSIGroupType = errors.New("invalid bsigroup type") + ErrInvalidBSIGroupRange = errors.New("invalid bsigroup range") + ErrInvalidBSIGroupValueType = errors.New("invalid bsigroup value type") + ErrBSIGroupValueTooLow = errors.New("bsigroup value too low") + ErrBSIGroupValueTooHigh = errors.New("bsigroup value too high") + ErrInvalidRangeOperation = errors.New("invalid range operation") + ErrInvalidBetweenValue = errors.New("invalid value for between operation") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") diff --git a/view.go b/view.go index 0d36faf2a..994a54579 100644 --- a/view.go +++ b/view.go @@ -31,7 +31,7 @@ import ( const ( ViewStandard = "standard" - ViewFieldPrefix = "field_" + viewBSIGroupPrefix = "bsig_" ) // IsValidView returns true if name is valid. @@ -98,7 +98,7 @@ func (v *View) Path() string { return v.path } func (v *View) Open() error { // Never keep a cache for field views. - if strings.HasPrefix(v.name, ViewFieldPrefix) { + if strings.HasPrefix(v.name, viewBSIGroupPrefix) { v.cacheType = CacheTypeNone } From a6ae39d0b08d5a6c1ffbc5c359fc8c9e644f6d34 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Jun 2018 17:18:34 -0500 Subject: [PATCH 20/78] remove cases of Field from frame.go --- api.go | 19 +-- broadcast.go | 20 +-- client_test.go | 10 +- executor.go | 4 +- executor_test.go | 4 +- frame.go | 62 ++++----- frame_internal_test.go | 4 +- frame_test.go | 4 +- internal/private.pb.go | 276 ++++++++++++++++++++--------------------- internal/private.proto | 11 +- server.go | 8 +- 11 files changed, 211 insertions(+), 211 deletions(-) diff --git a/api.go b/api.go index 0dc153380..e34a9142b 100644 --- a/api.go +++ b/api.go @@ -507,10 +507,10 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str // Send the create bsigroup message to all nodes. err := api.Broadcaster.SendSync( - &internal.CreateFieldMessage{ - Index: indexName, - Frame: frameName, - Field: encodeField(bsig), + &internal.CreateBSIGroupMessage{ + Index: indexName, + Frame: frameName, + BSIGroup: encodeBSIGroup(bsig), }) if err != nil { api.Logger.Printf("problem sending CreateField message: %s", err) @@ -518,6 +518,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str return errors.Wrap(err, "sending CreateField message") } +// TODO: remove this from the API // DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { if err := api.validate(apiDeleteField); err != nil { @@ -531,16 +532,16 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str } // Delete field. - if err := f.DeleteField(fieldName); err != nil { + if err := f.deleteBSIGroupAndView(fieldName); err != nil { return errors.Wrap(err, "deleting field") } // Send the delete field message to all nodes. err := api.Broadcaster.SendSync( - &internal.DeleteFieldMessage{ - Index: indexName, - Frame: frameName, - Field: fieldName, + &internal.DeleteBSIGroupMessage{ + Index: indexName, + Frame: frameName, + BSIGroup: fieldName, }) if err != nil { api.Logger.Printf("problem sending DeleteField message: %s", err) diff --git a/broadcast.go b/broadcast.go index 77b76126d..0840e8c92 100644 --- a/broadcast.go +++ b/broadcast.go @@ -127,8 +127,8 @@ const ( MessageTypeDeleteFrame MessageTypeCreateView MessageTypeDeleteView - MessageTypeCreateField - MessageTypeDeleteField + MessageTypeCreateBSIGroup + MessageTypeDeleteBSIGroup MessageTypeClusterStatus MessageTypeResizeInstruction MessageTypeResizeInstructionComplete @@ -157,10 +157,10 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeCreateView case *internal.DeleteViewMessage: typ = MessageTypeDeleteView - case *internal.CreateFieldMessage: - typ = MessageTypeCreateField - case *internal.DeleteFieldMessage: - typ = MessageTypeDeleteField + case *internal.CreateBSIGroupMessage: + typ = MessageTypeCreateBSIGroup + case *internal.DeleteBSIGroupMessage: + typ = MessageTypeDeleteBSIGroup case *internal.ClusterStatus: typ = MessageTypeClusterStatus case *internal.ResizeInstruction: @@ -207,10 +207,10 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.CreateViewMessage{} case MessageTypeDeleteView: m = &internal.DeleteViewMessage{} - case MessageTypeCreateField: - m = &internal.CreateFieldMessage{} - case MessageTypeDeleteField: - m = &internal.DeleteFieldMessage{} + case MessageTypeCreateBSIGroup: + m = &internal.CreateBSIGroupMessage{} + case MessageTypeDeleteBSIGroup: + m = &internal.DeleteBSIGroupMessage{} case MessageTypeClusterStatus: m = &internal.ClusterStatus{} case MessageTypeResizeInstruction: diff --git a/client_test.go b/client_test.go index 898869f2b..2bb255382 100644 --- a/client_test.go +++ b/client_test.go @@ -276,7 +276,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Sum. - sum, cnt, err := frame.FieldSum(nil, fldName) + sum, cnt, err := frame.Sum(nil, fldName) if err != nil { t.Fatal(err) } @@ -285,7 +285,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Min. - min, cnt, err := frame.FieldMin(nil, fldName) + min, cnt, err := frame.Min(nil, fldName) if err != nil { t.Fatal(err) } @@ -294,11 +294,11 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Min with Filter. - filter, err := frame.FieldRange(fldName, pql.GT, 40) + filter, err := frame.Range(fldName, pql.GT, 40) if err != nil { t.Fatal(err) } - min, cnt, err = frame.FieldMin(filter, fldName) + min, cnt, err = frame.Min(filter, fldName) if err != nil { t.Fatal(err) } @@ -307,7 +307,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Max. - max, cnt, err := frame.FieldMax(nil, fldName) + max, cnt, err := frame.Max(nil, fldName) if err != nil { t.Fatal(err) } diff --git a/executor.go b/executor.go index 43b44c315..b5e19e5b7 100644 --- a/executor.go +++ b/executor.go @@ -835,7 +835,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * return nil, ErrBSIGroupNotFound } - baseValueMin, baseValueMax, outOfRange := field.BaseValueBetween(predicates[0], predicates[1]) + baseValueMin, baseValueMax, outOfRange := field.baseValueBetween(predicates[0], predicates[1]) if outOfRange { return NewRow(), nil } @@ -868,7 +868,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * return nil, ErrBSIGroupNotFound } - baseValue, outOfRange := field.BaseValue(cond.Op, value) + baseValue, outOfRange := field.baseValue(cond.Op, value) if outOfRange && cond.Op != pql.NEQ { return NewRow(), nil } diff --git a/executor_test.go b/executor_test.go index 3429529ee..578ddc11b 100644 --- a/executor_test.go +++ b/executor_test.go @@ -290,7 +290,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } f := hldr.Frame("i", "f") - if value, exists, err := f.FieldValue(10, "f"); err != nil { + if value, exists, err := f.Value(10, "f"); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") @@ -298,7 +298,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Fatalf("unexpected value: %v", value) } - if value, exists, err := f.FieldValue(100, "f"); err != nil { + if value, exists, err := f.Value(100, "f"); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") diff --git a/frame.go b/frame.go index 6e2ef0bf7..65d98937c 100644 --- a/frame.go +++ b/frame.go @@ -433,8 +433,8 @@ func (f *Frame) addBSIGroup(bsig *bsiGroup) error { } // TODO: merge this into the un-exported deleteBSIGroup. -// DeleteField deletes an existing field on the schema. -func (f *Frame) DeleteField(name string) error { +// deleteBSIGroupAndView deletes an existing field on the schema. +func (f *Frame) deleteBSIGroupAndView(name string) error { f.mu.Lock() defer f.mu.Unlock() @@ -705,8 +705,8 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change return changed, nil } -// FieldValue reads a field value for a column. -func (f *Frame) FieldValue(columnID uint64, name string) (value int64, exists bool, err error) { +// Value reads a bsiGroup value for a column. +func (f *Frame) Value(columnID uint64, name string) (value int64, exists bool, err error) { field := f.bsiGroup(name) if field == nil { return 0, false, ErrBSIGroupNotFound @@ -751,9 +751,9 @@ func (f *Frame) SetValue(columnID uint64, value int64) (changed bool, err error) return view.setValue(columnID, field.BitDepth(), baseValue) } -// FieldSum returns the sum and count for a field. +// Sum returns the sum and count for a field. // An optional filtering row can be provided. -func (f *Frame) FieldSum(filter *Row, name string) (sum, count int64, err error) { +func (f *Frame) Sum(filter *Row, name string) (sum, count int64, err error) { field := f.bsiGroup(name) if field == nil { return 0, 0, ErrBSIGroupNotFound @@ -771,9 +771,9 @@ func (f *Frame) FieldSum(filter *Row, name string) (sum, count int64, err error) return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil } -// FieldMin returns the min for a bsiGroup. +// Min returns the min for a field. // An optional filtering row can be provided. -func (f *Frame) FieldMin(filter *Row, name string) (min, count int64, err error) { +func (f *Frame) Min(filter *Row, name string) (min, count int64, err error) { bsig := f.bsiGroup(name) if bsig == nil { return 0, 0, ErrBSIGroupNotFound @@ -791,9 +791,9 @@ func (f *Frame) FieldMin(filter *Row, name string) (min, count int64, err error) return int64(vmin) + bsig.Min, int64(vcount), nil } -// FieldMax returns the max for a field. +// Max returns the max for a field. // An optional filtering row can be provided. -func (f *Frame) FieldMax(filter *Row, name string) (max, count int64, err error) { +func (f *Frame) Max(filter *Row, name string) (max, count int64, err error) { field := f.bsiGroup(name) if field == nil { return 0, 0, ErrBSIGroupNotFound @@ -811,7 +811,7 @@ func (f *Frame) FieldMax(filter *Row, name string) (max, count int64, err error) return int64(vmax) + field.Min, int64(vcount), nil } -func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Row, error) { +func (f *Frame) Range(name string, op pql.Token, predicate int64) (*Row, error) { // Retrieve and validate field. field := f.bsiGroup(name) if field == nil { @@ -826,7 +826,7 @@ func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Row, er return nil, nil } - baseValue, outOfRange := field.BaseValue(op, predicate) + baseValue, outOfRange := field.baseValue(op, predicate) if outOfRange { return NewRow(), nil } @@ -834,7 +834,7 @@ func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Row, er return view.FieldRange(op, field.BitDepth(), baseValue) } -func (f *Frame) FieldRangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { +func (f *Frame) RangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { // Retrieve and validate field. field := f.bsiGroup(name) if field == nil { @@ -849,7 +849,7 @@ func (f *Frame) FieldRangeBetween(name string, predicateMin, predicateMax int64) return nil, nil } - baseValueMin, baseValueMax, outOfRange := field.BaseValueBetween(predicateMin, predicateMax) + baseValueMin, baseValueMax, outOfRange := field.baseValueBetween(predicateMin, predicateMax) if outOfRange { return NewRow(), nil } @@ -917,23 +917,23 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // ImportValue bulk imports range-encoded value data. func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { viewName := viewBSIGroupPrefix + fieldName - // Get the field so we know bitDepth. - field := f.bsiGroup(fieldName) - if field == nil { - return fmt.Errorf("Field does not exist: %s", fieldName) + // Get the bsiGroup so we know bitDepth. + bsig := f.bsiGroup(fieldName) + if bsig == nil { + return errors.Wrap(ErrBSIGroupNotFound, fieldName) } // Split import data by fragment. dataByFragment := make(map[importKey]importValueData) for i := range columnIDs { columnID, value := columnIDs[i], values[i] - if int64(value) > field.Max { + if int64(value) > bsig.Max { return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooHigh, columnID, value) - } else if int64(value) < field.Min { + } else if int64(value) < bsig.Min { return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooLow, columnID, value) } - // Attach value to each field view. + // Attach value to each bsiGroup view. for _, name := range []string{viewName} { key := importKey{View: name, Slice: columnID / SliceWidth} data := dataByFragment[key] @@ -960,10 +960,10 @@ func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64 baseValues := make([]uint64, len(data.Values)) for i, value := range data.Values { - baseValues[i] = uint64(value - field.Min) + baseValues[i] = uint64(value - bsig.Min) } - if err := frag.ImportValue(data.ColumnIDs, baseValues, field.BitDepth()); err != nil { + if err := frag.ImportValue(data.ColumnIDs, baseValues, bsig.BitDepth()); err != nil { return err } } @@ -1103,19 +1103,19 @@ func (b *bsiGroup) BitDepth() uint { return 63 } -// BaseValue adjusts the value to align with the range for Field for a certain +// baseValue adjusts the value to align with the range for Field for a certain // operation type. // Note: There is an edge case for GT and LT where this returns a baseValue // that does not fully encompass the range. // ex: Field.Min = 0, Field.Max = 1023 -// BaseValue(LT, 2000) returns 1023, which will perform "LT 1023" and effectively +// baseValue(LT, 2000) returns 1023, which will perform "LT 1023" and effectively // exclude any columns with value = 1023. // Note that in this case (because the range uses the full BitDepth 0 to 1023), // we can't simply return 1024. // In order to make this work, we effectively need to change the operator to LTE. // Executor.executeFieldRangeSlice() takes this into account and returns // `frag.FieldNotNull(field.BitDepth())` in such instances. -func (b *bsiGroup) BaseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { +func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { if op == pql.GT || op == pql.GTE { if value > b.Max { return baseValue, true @@ -1139,8 +1139,8 @@ func (b *bsiGroup) BaseValue(op pql.Token, value int64) (baseValue uint64, outOf return baseValue, false } -// BaseValueBetween adjusts the min/max value to align with the range for Field. -func (b *bsiGroup) BaseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { +// baseValueBetween adjusts the min/max value to align with the range for Field. +func (b *bsiGroup) baseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { if max < b.Min || min > b.Max { return baseValueMin, baseValueMax, true } @@ -1168,11 +1168,11 @@ func (b *bsiGroup) validate() error { return nil } -func encodeField(b *bsiGroup) *internal.Field { +func encodeBSIGroup(b *bsiGroup) *internal.BSIGroup { if b == nil { return nil } - return &internal.Field{ + return &internal.BSIGroup{ Name: b.Name, Type: b.Type, Min: int64(b.Min), @@ -1180,7 +1180,7 @@ func encodeField(b *bsiGroup) *internal.Field { } } -func decodeField(b *internal.Field) *bsiGroup { +func decodeBSIGroup(b *internal.BSIGroup) *bsiGroup { if b == nil { return nil } diff --git a/frame_internal_test.go b/frame_internal_test.go index 749e4e720..5cba2e84b 100644 --- a/frame_internal_test.go +++ b/frame_internal_test.go @@ -101,7 +101,7 @@ func TestField_BaseValue(t *testing.T) { {b2, pql.EQ, 105, 5, false}, {b2, pql.EQ, 1105, 0, true}, } { - bv, oor := tt.f.BaseValue(tt.op, tt.val) + bv, oor := tt.f.baseValue(tt.op, tt.val) if oor != tt.expOutOfRange { t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor) } else if !reflect.DeepEqual(bv, tt.expBaseValue) { @@ -138,7 +138,7 @@ func TestField_BaseValue(t *testing.T) { {b2, 120, 1105, 20, 1000, false}, {b2, 1105, 2000, 0, 0, true}, } { - min, max, oor := tt.f.BaseValueBetween(tt.predMin, tt.predMax) + min, max, oor := tt.f.baseValueBetween(tt.predMin, tt.predMax) if oor != tt.expOutOfRange { t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor) } else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) { diff --git a/frame_test.go b/frame_test.go index 6cc75f6be..aeae2b7ba 100644 --- a/frame_test.go +++ b/frame_test.go @@ -94,7 +94,7 @@ func TestFrame_SetValue(t *testing.T) { } // Read value. - if value, exists, err := f.FieldValue(100, "f"); err != nil { + if value, exists, err := f.Value(100, "f"); err != nil { t.Fatal(err) } else if value != 21 { t.Fatalf("unexpected value: %d", value) @@ -138,7 +138,7 @@ func TestFrame_SetValue(t *testing.T) { } // Read value. - if value, exists, err := f.FieldValue(100, "f"); err != nil { + if value, exists, err := f.Value(100, "f"); err != nil { t.Fatal(err) } else if value != 23 { t.Fatalf("unexpected value: %d", value) diff --git a/internal/private.pb.go b/internal/private.pb.go index 20909445c..4e1aa8589 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -21,8 +21,8 @@ CreateIndexMessage CreateFrameMessage DeleteFrameMessage - CreateFieldMessage - DeleteFieldMessage + CreateBSIGroupMessage + DeleteBSIGroupMessage Frame Schema Index @@ -32,7 +32,7 @@ NodeEventMessage NodeStatus ClusterStatus - Field + BSIGroup CreateViewMessage DeleteViewMessage ResizeInstruction @@ -366,66 +366,66 @@ func (m *DeleteFrameMessage) GetFrame() string { return "" } -type CreateFieldMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Field *Field `protobuf:"bytes,3,opt,name=Field" json:"Field,omitempty"` +type CreateBSIGroupMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + BSIGroup *BSIGroup `protobuf:"bytes,3,opt,name=BSIGroup" json:"BSIGroup,omitempty"` } -func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } -func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } -func (*CreateFieldMessage) ProtoMessage() {} -func (*CreateFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } +func (m *CreateBSIGroupMessage) Reset() { *m = CreateBSIGroupMessage{} } +func (m *CreateBSIGroupMessage) String() string { return proto.CompactTextString(m) } +func (*CreateBSIGroupMessage) ProtoMessage() {} +func (*CreateBSIGroupMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } -func (m *CreateFieldMessage) GetIndex() string { +func (m *CreateBSIGroupMessage) GetIndex() string { if m != nil { return m.Index } return "" } -func (m *CreateFieldMessage) GetFrame() string { +func (m *CreateBSIGroupMessage) GetFrame() string { if m != nil { return m.Frame } return "" } -func (m *CreateFieldMessage) GetField() *Field { +func (m *CreateBSIGroupMessage) GetBSIGroup() *BSIGroup { if m != nil { - return m.Field + return m.BSIGroup } return nil } -type DeleteFieldMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` +type DeleteBSIGroupMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + BSIGroup string `protobuf:"bytes,3,opt,name=BSIGroup,proto3" json:"BSIGroup,omitempty"` } -func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } -func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteFieldMessage) ProtoMessage() {} -func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } +func (m *DeleteBSIGroupMessage) Reset() { *m = DeleteBSIGroupMessage{} } +func (m *DeleteBSIGroupMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteBSIGroupMessage) ProtoMessage() {} +func (*DeleteBSIGroupMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } -func (m *DeleteFieldMessage) GetIndex() string { +func (m *DeleteBSIGroupMessage) GetIndex() string { if m != nil { return m.Index } return "" } -func (m *DeleteFieldMessage) GetFrame() string { +func (m *DeleteBSIGroupMessage) GetFrame() string { if m != nil { return m.Frame } return "" } -func (m *DeleteFieldMessage) GetField() string { +func (m *DeleteBSIGroupMessage) GetBSIGroup() string { if m != nil { - return m.Field + return m.BSIGroup } return "" } @@ -678,40 +678,40 @@ func (m *ClusterStatus) GetNodes() []*Node { return nil } -type Field struct { +type BSIGroup struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` Min int64 `protobuf:"varint,3,opt,name=Min,proto3" json:"Min,omitempty"` Max int64 `protobuf:"varint,4,opt,name=Max,proto3" json:"Max,omitempty"` } -func (m *Field) Reset() { *m = Field{} } -func (m *Field) String() string { return proto.CompactTextString(m) } -func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (m *BSIGroup) Reset() { *m = BSIGroup{} } +func (m *BSIGroup) String() string { return proto.CompactTextString(m) } +func (*BSIGroup) ProtoMessage() {} +func (*BSIGroup) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } -func (m *Field) GetName() string { +func (m *BSIGroup) GetName() string { if m != nil { return m.Name } return "" } -func (m *Field) GetType() string { +func (m *BSIGroup) GetType() string { if m != nil { return m.Type } return "" } -func (m *Field) GetMin() int64 { +func (m *BSIGroup) GetMin() int64 { if m != nil { return m.Min } return 0 } -func (m *Field) GetMax() int64 { +func (m *BSIGroup) GetMax() int64 { if m != nil { return m.Max } @@ -997,8 +997,8 @@ func init() { proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") - proto.RegisterType((*CreateFieldMessage)(nil), "internal.CreateFieldMessage") - proto.RegisterType((*DeleteFieldMessage)(nil), "internal.DeleteFieldMessage") + proto.RegisterType((*CreateBSIGroupMessage)(nil), "internal.CreateBSIGroupMessage") + proto.RegisterType((*DeleteBSIGroupMessage)(nil), "internal.DeleteBSIGroupMessage") proto.RegisterType((*Frame)(nil), "internal.Frame") proto.RegisterType((*Schema)(nil), "internal.Schema") proto.RegisterType((*Index)(nil), "internal.Index") @@ -1008,7 +1008,7 @@ func init() { proto.RegisterType((*NodeEventMessage)(nil), "internal.NodeEventMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") - proto.RegisterType((*Field)(nil), "internal.Field") + proto.RegisterType((*BSIGroup)(nil), "internal.BSIGroup") proto.RegisterType((*CreateViewMessage)(nil), "internal.CreateViewMessage") proto.RegisterType((*DeleteViewMessage)(nil), "internal.DeleteViewMessage") proto.RegisterType((*ResizeInstruction)(nil), "internal.ResizeInstruction") @@ -1436,7 +1436,7 @@ func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *CreateFieldMessage) Marshal() (dAtA []byte, err error) { +func (m *CreateBSIGroupMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1446,7 +1446,7 @@ func (m *CreateFieldMessage) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { +func (m *CreateBSIGroupMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1463,11 +1463,11 @@ func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) i += copy(dAtA[i:], m.Frame) } - if m.Field != nil { + if m.BSIGroup != nil { dAtA[i] = 0x1a i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Field.Size())) - n9, err := m.Field.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.BSIGroup.Size())) + n9, err := m.BSIGroup.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -1476,7 +1476,7 @@ func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *DeleteFieldMessage) Marshal() (dAtA []byte, err error) { +func (m *DeleteBSIGroupMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1486,7 +1486,7 @@ func (m *DeleteFieldMessage) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { +func (m *DeleteBSIGroupMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1503,11 +1503,11 @@ func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) i += copy(dAtA[i:], m.Frame) } - if len(m.Field) > 0 { + if len(m.BSIGroup) > 0 { dAtA[i] = 0x1a i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.BSIGroup))) + i += copy(dAtA[i:], m.BSIGroup) } return i, nil } @@ -1859,7 +1859,7 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Field) Marshal() (dAtA []byte, err error) { +func (m *BSIGroup) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1869,7 +1869,7 @@ func (m *Field) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Field) MarshalTo(dAtA []byte) (int, error) { +func (m *BSIGroup) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -2459,7 +2459,7 @@ func (m *DeleteFrameMessage) Size() (n int) { return n } -func (m *CreateFieldMessage) Size() (n int) { +func (m *CreateBSIGroupMessage) Size() (n int) { var l int _ = l l = len(m.Index) @@ -2470,14 +2470,14 @@ func (m *CreateFieldMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Field != nil { - l = m.Field.Size() + if m.BSIGroup != nil { + l = m.BSIGroup.Size() n += 1 + l + sovPrivate(uint64(l)) } return n } -func (m *DeleteFieldMessage) Size() (n int) { +func (m *DeleteBSIGroupMessage) Size() (n int) { var l int _ = l l = len(m.Index) @@ -2488,7 +2488,7 @@ func (m *DeleteFieldMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Field) + l = len(m.BSIGroup) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2642,7 +2642,7 @@ func (m *ClusterStatus) Size() (n int) { return n } -func (m *Field) Size() (n int) { +func (m *BSIGroup) Size() (n int) { var l int _ = l l = len(m.Name) @@ -4304,7 +4304,7 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { +func (m *CreateBSIGroupMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -4327,10 +4327,10 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: CreateFieldMessage: wiretype end group for non-group") + return fmt.Errorf("proto: CreateBSIGroupMessage: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: CreateFieldMessage: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: CreateBSIGroupMessage: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -4393,7 +4393,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field BSIGroup", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -4417,10 +4417,10 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.Field == nil { - m.Field = &Field{} + if m.BSIGroup == nil { + m.BSIGroup = &BSIGroup{} } - if err := m.Field.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.BSIGroup.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -4445,7 +4445,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { +func (m *DeleteBSIGroupMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -4468,10 +4468,10 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: DeleteFieldMessage: wiretype end group for non-group") + return fmt.Errorf("proto: DeleteBSIGroupMessage: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: DeleteFieldMessage: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: DeleteBSIGroupMessage: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -4534,7 +4534,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field BSIGroup", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -4559,7 +4559,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Field = string(dAtA[iNdEx:postIndex]) + m.BSIGroup = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex @@ -5671,7 +5671,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } return nil } -func (m *Field) Unmarshal(dAtA []byte) error { +func (m *BSIGroup) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -5694,10 +5694,10 @@ func (m *Field) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Field: wiretype end group for non-group") + return fmt.Errorf("proto: BSIGroup: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Field: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: BSIGroup: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -7075,70 +7075,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1029 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1b, 0x45, - 0x18, 0x67, 0xbd, 0x6b, 0x27, 0xfe, 0x82, 0xd3, 0x64, 0x5a, 0xc2, 0x16, 0xa1, 0x60, 0x46, 0x45, - 0x18, 0x0e, 0x51, 0x69, 0x2f, 0xbc, 0x2a, 0x45, 0xb1, 0x83, 0x58, 0x44, 0x22, 0x98, 0x4d, 0x7a, - 0x40, 0xe2, 0x30, 0xb5, 0x47, 0xe9, 0x2a, 0xeb, 0x1d, 0xb3, 0x3b, 0x9b, 0xc4, 0x3d, 0x70, 0x85, - 0x0b, 0x77, 0xc4, 0x8d, 0xff, 0x86, 0x23, 0x7f, 0x02, 0x0a, 0xff, 0x08, 0x9a, 0x6f, 0x66, 0x1f, - 0xf1, 0xa3, 0xa9, 0x4c, 0x6f, 0xf3, 0xbd, 0x5f, 0xbf, 0x6f, 0x66, 0xa0, 0x33, 0x49, 0xa3, 0x0b, - 0xae, 0xc4, 0xde, 0x24, 0x95, 0x4a, 0x92, 0xf5, 0x28, 0x51, 0x22, 0x4d, 0x78, 0x4c, 0x37, 0xa0, - 0x1d, 0x24, 0x23, 0x71, 0x75, 0x24, 0x14, 0xa7, 0x7f, 0x3a, 0xd0, 0xfe, 0x2a, 0xe5, 0x63, 0xa1, - 0x29, 0xf2, 0x2e, 0xb4, 0xfb, 0x7c, 0xf8, 0x5c, 0x9c, 0x4c, 0x27, 0xc2, 0x77, 0xbb, 0x4e, 0xaf, - 0xcd, 0x2a, 0x46, 0x29, 0x0d, 0xa3, 0x17, 0xc2, 0xf7, 0xba, 0x4e, 0xaf, 0xc3, 0x2a, 0x06, 0xe9, - 0xc2, 0xc6, 0x49, 0x34, 0x16, 0xdf, 0xe7, 0x3c, 0x51, 0xf9, 0xd8, 0x6f, 0xa2, 0x75, 0x9d, 0x45, - 0x08, 0x78, 0xe8, 0x78, 0x1d, 0x45, 0x78, 0x26, 0x5b, 0xe0, 0x1e, 0x45, 0x89, 0xdf, 0xee, 0x3a, - 0x3d, 0x97, 0xe9, 0x23, 0x72, 0xf8, 0x95, 0x0f, 0x96, 0xc3, 0xaf, 0x28, 0x85, 0xcd, 0x60, 0x3c, - 0x91, 0xa9, 0x62, 0x22, 0x9b, 0xc8, 0x24, 0x43, 0xab, 0xc3, 0x34, 0xf5, 0x1d, 0x74, 0xa4, 0x8f, - 0xf4, 0x67, 0xd8, 0x3a, 0x88, 0xe5, 0xf0, 0x7c, 0xc0, 0x15, 0x67, 0xe2, 0xa7, 0x5c, 0x64, 0x8a, - 0xdc, 0x83, 0x26, 0x16, 0x6a, 0xf5, 0x0c, 0xa1, 0xb9, 0x58, 0xb0, 0xdf, 0x30, 0x5c, 0x24, 0x34, - 0x17, 0xed, 0xb1, 0x6a, 0x8f, 0x19, 0x42, 0x73, 0xc3, 0x38, 0x1a, 0x9a, 0x6a, 0x3d, 0x66, 0x08, - 0x5d, 0xc7, 0xd3, 0x48, 0x5c, 0xda, 0x12, 0xf1, 0x4c, 0x03, 0xd8, 0xae, 0xc5, 0xb7, 0x69, 0xee, - 0x40, 0x8b, 0xc9, 0xcb, 0x60, 0x90, 0xf9, 0x4e, 0xd7, 0xed, 0x79, 0xcc, 0x52, 0xd8, 0x48, 0x19, - 0xe7, 0xe3, 0x44, 0x8b, 0x1a, 0x28, 0xaa, 0x18, 0xf4, 0x3e, 0x34, 0xb1, 0xab, 0xba, 0xca, 0xca, - 0x56, 0x1f, 0xe9, 0x2f, 0x0e, 0xb4, 0x8f, 0xf8, 0x15, 0xa6, 0x91, 0x91, 0x27, 0xb0, 0x1e, 0x2a, - 0x9e, 0x8c, 0x78, 0x3a, 0x42, 0xa5, 0x8d, 0x47, 0xef, 0xef, 0x15, 0x53, 0xde, 0x2b, 0xd5, 0xf6, - 0x0a, 0x9d, 0xc3, 0x44, 0xa5, 0x53, 0x56, 0x9a, 0xbc, 0xf3, 0x05, 0x74, 0x6e, 0x88, 0x74, 0xbc, - 0x73, 0x31, 0x2d, 0xba, 0x7a, 0x2e, 0xa6, 0xba, 0xfe, 0x0b, 0x1e, 0xe7, 0xa6, 0x57, 0x1e, 0x33, - 0xc4, 0xe7, 0x8d, 0x4f, 0x1d, 0xba, 0x0f, 0xa4, 0x9f, 0x0a, 0xae, 0x04, 0x06, 0x39, 0x12, 0x59, - 0xc6, 0xcf, 0xc4, 0xf2, 0x8e, 0x9b, 0x2e, 0x36, 0x6a, 0x5d, 0xa4, 0x1f, 0x03, 0x19, 0x88, 0x58, - 0x28, 0x61, 0xc1, 0xf8, 0x12, 0x0f, 0x34, 0x2c, 0xa2, 0xdd, 0xae, 0x4b, 0x3e, 0x04, 0x4f, 0x63, - 0x19, 0x83, 0x6d, 0x3c, 0xba, 0x5b, 0x75, 0xa4, 0x04, 0x3d, 0x43, 0x05, 0x1a, 0x15, 0x4e, 0x2d, - 0xfe, 0x6f, 0x29, 0x61, 0x01, 0x68, 0x8a, 0x50, 0xee, 0x6c, 0xa8, 0x72, 0xa3, 0x6c, 0xa8, 0xfd, - 0xa2, 0xd6, 0x55, 0x43, 0xd1, 0xb3, 0x32, 0xd9, 0x48, 0xc4, 0xa3, 0x55, 0x92, 0xfd, 0x00, 0x9a, - 0x68, 0x6b, 0xb3, 0xbd, 0x53, 0xcb, 0x56, 0xb3, 0x99, 0x91, 0xd2, 0xa7, 0x65, 0xaa, 0xab, 0x06, - 0xba, 0x57, 0x0f, 0xd4, 0x2e, 0xfc, 0xfe, 0x60, 0x75, 0xf5, 0xf6, 0x1c, 0x6b, 0x1b, 0xe3, 0x09, - 0xcf, 0xcb, 0x67, 0x36, 0xd3, 0x48, 0xed, 0x5b, 0xaf, 0x5b, 0xe6, 0xbb, 0x5d, 0x57, 0xfb, 0x46, - 0x82, 0x3e, 0x86, 0x56, 0x38, 0x7c, 0x2e, 0xc6, 0x9c, 0x7c, 0x04, 0x6b, 0x98, 0x9a, 0xc8, 0xec, - 0x46, 0xdc, 0x99, 0x99, 0x3f, 0x2b, 0xe4, 0x74, 0x60, 0x4b, 0x5a, 0x92, 0x50, 0x0b, 0x43, 0x67, - 0xbe, 0x37, 0xeb, 0x06, 0xf9, 0xcc, 0x8a, 0xe9, 0x21, 0xb8, 0xa7, 0x2c, 0xd0, 0x9b, 0x8e, 0x19, - 0x14, 0x5e, 0x2c, 0xa5, 0x7d, 0x7f, 0x2d, 0x33, 0x65, 0x1b, 0x84, 0x67, 0xcd, 0xfb, 0x4e, 0xa6, - 0x0a, 0xdb, 0xd3, 0x61, 0x78, 0xa6, 0x3f, 0x82, 0x77, 0x2c, 0x47, 0x82, 0x6c, 0x42, 0x23, 0x18, - 0x58, 0x1f, 0x8d, 0x60, 0x40, 0xde, 0x43, 0xf7, 0xb6, 0x2f, 0x9d, 0x2a, 0x89, 0x53, 0x16, 0x30, - 0x0c, 0xfc, 0x00, 0x3a, 0x41, 0xd6, 0x97, 0x32, 0x1d, 0x45, 0x09, 0x57, 0x32, 0x45, 0xaf, 0xeb, - 0xec, 0x26, 0x93, 0xee, 0xc3, 0x96, 0x76, 0x1f, 0x2a, 0xae, 0x4a, 0xf4, 0xed, 0x40, 0x4b, 0xf3, - 0xca, 0x70, 0x96, 0xc2, 0x6d, 0xd5, 0x7a, 0xc5, 0x50, 0x91, 0xa0, 0xdf, 0x1a, 0x0f, 0x87, 0x17, - 0x22, 0x51, 0x35, 0x50, 0x20, 0x8d, 0x0e, 0x3a, 0xcc, 0x10, 0x84, 0x9a, 0x52, 0x6c, 0xce, 0x9b, - 0x55, 0xce, 0x9a, 0xcb, 0x50, 0x46, 0x7f, 0x73, 0x00, 0x8a, 0x84, 0xf2, 0xac, 0x34, 0x71, 0x96, - 0x9b, 0x90, 0x4f, 0x6a, 0x37, 0xdf, 0x3c, 0x4e, 0x4a, 0x11, 0xab, 0xdd, 0x8f, 0xbd, 0x02, 0x16, - 0x16, 0xf2, 0x5b, 0x95, 0xbe, 0xe1, 0xdb, 0x31, 0xe9, 0xab, 0xa0, 0xd3, 0x8f, 0xf3, 0x4c, 0x89, - 0xd4, 0x66, 0xa4, 0x6f, 0x68, 0xc3, 0x28, 0xfb, 0x53, 0x31, 0x16, 0xb7, 0x88, 0x3c, 0x80, 0xa6, - 0xce, 0xd4, 0x60, 0x73, 0xbe, 0x0c, 0x23, 0xa4, 0xa1, 0xdd, 0x8e, 0x85, 0xb0, 0x2b, 0x5e, 0xc8, - 0xc6, 0xfc, 0x0b, 0xe9, 0xce, 0xbd, 0x90, 0x5e, 0xf5, 0x42, 0x86, 0xb0, 0x6d, 0x6e, 0x07, 0xbd, - 0x0f, 0xab, 0xec, 0x6c, 0xf1, 0xa4, 0xb9, 0xb5, 0x27, 0x2d, 0x84, 0x6d, 0x73, 0x13, 0xbc, 0x4e, - 0xa7, 0x7f, 0x34, 0x60, 0x9b, 0x89, 0x2c, 0x7a, 0x21, 0x82, 0x24, 0x53, 0x69, 0x3e, 0x54, 0x91, - 0x4c, 0xb4, 0xfd, 0x37, 0xf2, 0x99, 0x6d, 0xb5, 0xcb, 0x0c, 0xf1, 0x2a, 0x48, 0x22, 0x0f, 0x61, - 0x63, 0x16, 0xfd, 0xf3, 0xaa, 0x75, 0x15, 0xf2, 0x10, 0xd6, 0x42, 0x99, 0xa7, 0xc3, 0x72, 0xb7, - 0x77, 0x2a, 0x6d, 0x93, 0x99, 0x11, 0xb3, 0x42, 0xad, 0x86, 0xa3, 0xe6, 0xcb, 0x71, 0x44, 0x9e, - 0xcc, 0xe0, 0xc8, 0x6f, 0xa1, 0xc1, 0xdb, 0x95, 0xc1, 0x0d, 0x31, 0xbb, 0xa9, 0x4d, 0x7f, 0x75, - 0xe0, 0xcd, 0x7a, 0x0a, 0xaf, 0xb4, 0x18, 0xe5, 0x44, 0x1a, 0x0b, 0x27, 0xe2, 0x2e, 0x9a, 0x88, - 0x57, 0x4d, 0xa4, 0x7a, 0x9d, 0x9b, 0xf5, 0xd7, 0xf9, 0x1c, 0xee, 0xcf, 0x8d, 0xa9, 0x2f, 0xc7, - 0x13, 0x8d, 0x87, 0xff, 0x31, 0x2e, 0x7d, 0x65, 0xa4, 0xa9, 0x1d, 0x54, 0x9b, 0x19, 0x82, 0x7e, - 0x06, 0x6f, 0x85, 0x42, 0xd5, 0x86, 0x54, 0xa0, 0xad, 0x0b, 0xee, 0xb1, 0xb8, 0x5c, 0x52, 0xbe, - 0x16, 0xd1, 0x2f, 0xc1, 0x3f, 0x9d, 0x8c, 0xb8, 0x12, 0x2b, 0x59, 0x1f, 0xc0, 0xfa, 0x89, 0x9c, - 0xc8, 0x58, 0x9e, 0x4d, 0x6f, 0x59, 0x79, 0x1f, 0xd6, 0xcc, 0xfd, 0x68, 0x3e, 0x6c, 0x6d, 0x56, - 0x90, 0xf4, 0xae, 0x06, 0xf4, 0x90, 0xc7, 0xc3, 0x3c, 0xd6, 0x69, 0xe8, 0x9f, 0x5b, 0x76, 0xb0, - 0xf5, 0xd7, 0xf5, 0xae, 0xf3, 0xf7, 0xf5, 0xae, 0xf3, 0xcf, 0xf5, 0xae, 0xf3, 0xfb, 0xbf, 0xbb, - 0x6f, 0x3c, 0x6b, 0xe1, 0x37, 0xfc, 0xf1, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x71, 0x97, 0x84, - 0xb1, 0x97, 0x0b, 0x00, 0x00, + // 1038 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0x1b, 0xc5, + 0x1b, 0xff, 0xaf, 0x77, 0xed, 0xd8, 0x4f, 0xfe, 0x0e, 0xc9, 0x94, 0x84, 0x6d, 0x85, 0x82, 0x19, + 0x55, 0x22, 0x70, 0xb0, 0x4a, 0x7b, 0xe1, 0xad, 0x52, 0x14, 0x3b, 0xc0, 0x22, 0x12, 0xc1, 0x6c, + 0xd2, 0x03, 0x12, 0x42, 0x53, 0x7b, 0xd4, 0xae, 0xb2, 0xde, 0x31, 0xbb, 0xb3, 0x49, 0xdc, 0x03, + 0x57, 0xb8, 0x70, 0x47, 0xdc, 0xf8, 0x36, 0x1c, 0xf9, 0x08, 0x28, 0x7c, 0x11, 0x34, 0xcf, 0xcc, + 0xbe, 0xc4, 0x2f, 0x4d, 0x95, 0x72, 0x9b, 0xe7, 0xfd, 0xed, 0xf7, 0xcc, 0x0c, 0x74, 0xa7, 0x69, + 0x74, 0xce, 0x95, 0xe8, 0x4f, 0x53, 0xa9, 0x24, 0x69, 0x47, 0x89, 0x12, 0x69, 0xc2, 0x63, 0xba, + 0x0e, 0x9d, 0x20, 0x19, 0x8b, 0xcb, 0x23, 0xa1, 0x38, 0xfd, 0xc3, 0x81, 0xce, 0xe7, 0x29, 0x9f, + 0x08, 0x4d, 0x91, 0xb7, 0xa1, 0x33, 0xe0, 0xa3, 0xe7, 0xe2, 0x64, 0x36, 0x15, 0xbe, 0xdb, 0x73, + 0xf6, 0x3a, 0xac, 0x62, 0x94, 0xd2, 0x30, 0x7a, 0x21, 0x7c, 0xaf, 0xe7, 0xec, 0x75, 0x59, 0xc5, + 0x20, 0x3d, 0x58, 0x3f, 0x89, 0x26, 0xe2, 0xdb, 0x9c, 0x27, 0x2a, 0x9f, 0xf8, 0x4d, 0xb4, 0xae, + 0xb3, 0x08, 0x01, 0x0f, 0x1d, 0xb7, 0x51, 0x84, 0x67, 0xb2, 0x09, 0xee, 0x51, 0x94, 0xf8, 0x9d, + 0x9e, 0xb3, 0xe7, 0x32, 0x7d, 0x44, 0x0e, 0xbf, 0xf4, 0xc1, 0x72, 0xf8, 0x25, 0xa5, 0xb0, 0x11, + 0x4c, 0xa6, 0x32, 0x55, 0x4c, 0x64, 0x53, 0x99, 0x64, 0x68, 0x75, 0x98, 0xa6, 0xbe, 0x83, 0x8e, + 0xf4, 0x91, 0xfe, 0x04, 0x9b, 0x07, 0xb1, 0x1c, 0x9d, 0x0d, 0xb9, 0xe2, 0x4c, 0xfc, 0x98, 0x8b, + 0x4c, 0x91, 0x37, 0xa1, 0x89, 0x85, 0x5a, 0x3d, 0x43, 0x68, 0x2e, 0x16, 0xec, 0x37, 0x0c, 0x17, + 0x09, 0xcd, 0x45, 0x7b, 0xac, 0xda, 0x63, 0x86, 0xd0, 0xdc, 0x30, 0x8e, 0x46, 0xa6, 0x5a, 0x8f, + 0x19, 0x42, 0xd7, 0xf1, 0x24, 0x12, 0x17, 0xb6, 0x44, 0x3c, 0xd3, 0x00, 0xb6, 0x6a, 0xf1, 0x6d, + 0x9a, 0x3b, 0xd0, 0x62, 0xf2, 0x22, 0x18, 0x66, 0xbe, 0xd3, 0x73, 0xf7, 0x3c, 0x66, 0x29, 0x6c, + 0xa4, 0x8c, 0xf3, 0x49, 0xa2, 0x45, 0x0d, 0x14, 0x55, 0x0c, 0x7a, 0x17, 0x9a, 0xd8, 0x55, 0x5d, + 0x65, 0x65, 0xab, 0x8f, 0xf4, 0x67, 0x07, 0x3a, 0x47, 0xfc, 0x12, 0xd3, 0xc8, 0xc8, 0x63, 0x68, + 0x87, 0x8a, 0x27, 0x63, 0x9e, 0x8e, 0x51, 0x69, 0xfd, 0xe1, 0xbb, 0xfd, 0x62, 0xca, 0xfd, 0x52, + 0xad, 0x5f, 0xe8, 0x1c, 0x26, 0x2a, 0x9d, 0xb1, 0xd2, 0xe4, 0xde, 0xa7, 0xd0, 0xbd, 0x26, 0xd2, + 0xf1, 0xce, 0xc4, 0xac, 0xe8, 0xea, 0x99, 0x98, 0xe9, 0xfa, 0xcf, 0x79, 0x9c, 0x9b, 0x5e, 0x79, + 0xcc, 0x10, 0x9f, 0x34, 0x3e, 0x72, 0xe8, 0x3e, 0x90, 0x41, 0x2a, 0xb8, 0x12, 0x18, 0xe4, 0x48, + 0x64, 0x19, 0x7f, 0x26, 0x56, 0x77, 0xdc, 0x74, 0xb1, 0x51, 0xeb, 0x22, 0xfd, 0x00, 0xc8, 0x50, + 0xc4, 0x42, 0x09, 0x0b, 0xc6, 0x97, 0x78, 0xa0, 0x61, 0x11, 0xed, 0x66, 0x5d, 0xf2, 0x1e, 0x78, + 0x1a, 0xcb, 0x18, 0x6c, 0xfd, 0xe1, 0x9d, 0xaa, 0x23, 0x25, 0xe8, 0x19, 0x2a, 0xd0, 0xa8, 0x70, + 0x6a, 0xf1, 0x7f, 0x43, 0x09, 0x4b, 0x40, 0x53, 0x84, 0x72, 0xe7, 0x43, 0x95, 0x1b, 0x65, 0x43, + 0xed, 0x17, 0xb5, 0xde, 0x36, 0x14, 0xcd, 0x60, 0xdb, 0x24, 0x7b, 0x10, 0x06, 0x5f, 0xa4, 0x32, + 0x9f, 0xde, 0x26, 0xdf, 0x3e, 0xb4, 0x0b, 0x73, 0x9b, 0x33, 0xa9, 0x72, 0x2e, 0x24, 0xac, 0xd4, + 0xa1, 0x3f, 0xc0, 0xb6, 0x49, 0xfb, 0x75, 0x82, 0xde, 0x9b, 0x0b, 0xda, 0xa9, 0x05, 0xf8, 0xce, + 0x5a, 0xe8, 0x95, 0x3a, 0xd6, 0x96, 0xc6, 0x1f, 0x9e, 0x57, 0x0f, 0x72, 0xae, 0xbb, 0x3a, 0xae, + 0xde, 0xc1, 0xcc, 0x77, 0x7b, 0xae, 0x8e, 0x8b, 0x04, 0x7d, 0x04, 0xad, 0x70, 0xf4, 0x5c, 0x4c, + 0x38, 0x79, 0x1f, 0xd6, 0x30, 0x41, 0x91, 0xd9, 0x35, 0x79, 0x63, 0x0e, 0x14, 0xac, 0x90, 0xd3, + 0xa1, 0x2d, 0x6c, 0x45, 0x42, 0x2d, 0x0c, 0x9d, 0xf9, 0xde, 0xbc, 0x1b, 0xe4, 0x33, 0x2b, 0xa6, + 0x87, 0xe0, 0x9e, 0xb2, 0x40, 0xaf, 0x3f, 0x66, 0x50, 0x78, 0xb1, 0x94, 0xf6, 0xfd, 0xa5, 0xcc, + 0x94, 0x6d, 0x13, 0x9e, 0x35, 0xef, 0x1b, 0x99, 0x2a, 0xec, 0x50, 0x97, 0xe1, 0x99, 0x7e, 0x0f, + 0xde, 0xb1, 0x1c, 0x0b, 0xb2, 0x01, 0x8d, 0x60, 0x68, 0x7d, 0x34, 0x82, 0x21, 0x79, 0x07, 0xdd, + 0xdb, 0xbe, 0x74, 0xab, 0x24, 0x4e, 0x59, 0xc0, 0x30, 0xf0, 0x7d, 0xe8, 0x06, 0xd9, 0x40, 0xca, + 0x74, 0x1c, 0x25, 0x5c, 0xc9, 0x14, 0xbd, 0xb6, 0xd9, 0x75, 0x26, 0xdd, 0x87, 0x4d, 0xed, 0x3e, + 0x54, 0x5c, 0x95, 0x90, 0xdc, 0x81, 0x96, 0xe6, 0x95, 0xe1, 0x2c, 0x85, 0x2b, 0xac, 0xf5, 0x8a, + 0xd1, 0x22, 0x41, 0xbf, 0x36, 0x1e, 0x0e, 0xcf, 0x45, 0xa2, 0x6a, 0xd0, 0x40, 0x1a, 0x1d, 0x74, + 0x99, 0x21, 0x08, 0x35, 0xa5, 0xd8, 0x9c, 0x37, 0xaa, 0x9c, 0x35, 0x97, 0xa1, 0x8c, 0xfe, 0xea, + 0x00, 0x14, 0x09, 0xe5, 0x59, 0x69, 0xe2, 0xac, 0x36, 0x21, 0x1f, 0xd6, 0xae, 0xc3, 0x45, 0x9c, + 0x94, 0x22, 0x56, 0xbb, 0x34, 0xf7, 0x0a, 0x58, 0xd8, 0x0d, 0xd8, 0xac, 0xf4, 0x0d, 0xdf, 0x8e, + 0x49, 0xdf, 0x0f, 0xdd, 0x41, 0x9c, 0x67, 0x4a, 0xa4, 0x36, 0x23, 0x7d, 0x6d, 0x1b, 0x46, 0xd9, + 0x9f, 0x8a, 0xb1, 0xbc, 0x45, 0xe4, 0x3e, 0x34, 0x75, 0xa6, 0x06, 0x9b, 0x8b, 0x65, 0x18, 0x21, + 0x7d, 0x52, 0xed, 0xc8, 0x52, 0xe4, 0x15, 0x2f, 0x67, 0x63, 0xf1, 0xe5, 0x74, 0x17, 0x5e, 0x4e, + 0xaf, 0x7a, 0x39, 0x43, 0xd8, 0x32, 0xb7, 0x86, 0x5e, 0x89, 0xdb, 0x2c, 0x6f, 0xf1, 0xd4, 0xb9, + 0xb5, 0xa7, 0x2e, 0x84, 0x2d, 0x73, 0x2b, 0xfc, 0x97, 0x4e, 0x7f, 0x6f, 0xc0, 0x16, 0x13, 0x59, + 0xf4, 0x42, 0x04, 0x49, 0xa6, 0xd2, 0x7c, 0xa4, 0x22, 0x99, 0x68, 0xfb, 0xaf, 0xe4, 0x53, 0xdb, + 0x6d, 0x97, 0x19, 0xe2, 0x55, 0xc0, 0x44, 0x1e, 0xc0, 0xfa, 0xfc, 0x02, 0x2c, 0xaa, 0xd6, 0x55, + 0xc8, 0x03, 0x58, 0x0b, 0x65, 0x9e, 0x8e, 0xca, 0xf5, 0xde, 0xa9, 0xb4, 0x4d, 0x66, 0x46, 0xcc, + 0x0a, 0xb5, 0x1a, 0x94, 0x9a, 0x2f, 0x87, 0x12, 0x79, 0x3c, 0x07, 0x25, 0xbf, 0x85, 0x06, 0x6f, + 0x55, 0x06, 0xd7, 0xc4, 0xec, 0xba, 0x36, 0xfd, 0xc5, 0x81, 0xff, 0xd7, 0x53, 0x78, 0xa5, 0xdd, + 0x28, 0x27, 0xd2, 0x58, 0x3a, 0x11, 0x77, 0xd9, 0x44, 0xbc, 0x6a, 0x22, 0xd5, 0xab, 0xdd, 0xac, + 0xbf, 0xda, 0x67, 0x70, 0x77, 0x61, 0x4c, 0x03, 0x39, 0x99, 0x6a, 0x3c, 0xbc, 0xc6, 0xb8, 0xf4, + 0xad, 0x91, 0xa6, 0x76, 0x50, 0x1d, 0x66, 0x08, 0xfa, 0x31, 0x6c, 0x87, 0x42, 0xd5, 0x86, 0x54, + 0xa0, 0xad, 0x07, 0xee, 0xb1, 0xb8, 0x58, 0x51, 0xbe, 0x16, 0xd1, 0xcf, 0xc0, 0x3f, 0x9d, 0x8e, + 0xb9, 0x12, 0xb7, 0xb2, 0x3e, 0x80, 0xf6, 0x89, 0x9c, 0xca, 0x58, 0x3e, 0x9b, 0xdd, 0xb0, 0xf5, + 0x3e, 0xac, 0x99, 0x2b, 0xd2, 0x7c, 0xe4, 0x3a, 0xac, 0x20, 0xe9, 0x1d, 0x0d, 0xe8, 0x11, 0x8f, + 0x47, 0x79, 0xac, 0xd3, 0xd0, 0x3f, 0xba, 0xec, 0x60, 0xf3, 0xcf, 0xab, 0x5d, 0xe7, 0xaf, 0xab, + 0x5d, 0xe7, 0xef, 0xab, 0x5d, 0xe7, 0xb7, 0x7f, 0x76, 0xff, 0xf7, 0xb4, 0x85, 0xdf, 0xf3, 0x47, + 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xa3, 0xc6, 0x83, 0xfe, 0xaf, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index b530257ae..df004456f 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -12,7 +12,6 @@ message FrameMeta { int64 Min = 9; int64 Max = 10; string TimeQuantum = 5; - //repeated Field Fields = 7; } message ImportResponse { @@ -65,16 +64,16 @@ message DeleteFrameMessage { string Frame = 2; } -message CreateFieldMessage { +message CreateBSIGroupMessage { string Index = 1; string Frame = 2; - Field Field = 3; + BSIGroup BSIGroup = 3; } -message DeleteFieldMessage { +message DeleteBSIGroupMessage { string Index = 1; string Frame = 2; - string Field = 3; + string BSIGroup = 3; } message Frame { @@ -126,7 +125,7 @@ message ClusterStatus { repeated Node Nodes = 3; } -message Field { +message BSIGroup { string Name = 1; string Type = 2; int64 Min = 3; diff --git a/server.go b/server.go index e7c566cbf..1e4f5a421 100644 --- a/server.go +++ b/server.go @@ -470,15 +470,15 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err := idx.DeleteFrame(obj.Frame); err != nil { return err } - case *internal.CreateFieldMessage: + case *internal.CreateBSIGroupMessage: f := s.Holder.Frame(obj.Index, obj.Frame) - field := decodeField(obj.Field) + field := decodeBSIGroup(obj.BSIGroup) if err := f.createBSIGroup(field); err != nil { return err } - case *internal.DeleteFieldMessage: + case *internal.DeleteBSIGroupMessage: f := s.Holder.Frame(obj.Index, obj.Frame) - if err := f.DeleteField(obj.Field); err != nil { + if err := f.deleteBSIGroupAndView(obj.BSIGroup); err != nil { return err } case *internal.CreateViewMessage: From d80d3b6d5121d35b2f43a9e7896c57030e82d440 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Jun 2018 17:25:57 -0500 Subject: [PATCH 21/78] rename cases of Field in view.go --- executor_test.go | 2 +- frame.go | 12 ++++++------ view.go | 24 ++++++++++++------------ 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/executor_test.go b/executor_test.go index 578ddc11b..4b75686a4 100644 --- a/executor_test.go +++ b/executor_test.go @@ -335,7 +335,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Run("ErrInvalidFieldValueType", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err != pilosa.ErrInvalidBSIGroupValueType { t.Fatalf("unexpected error: %s", err) } }) diff --git a/frame.go b/frame.go index 65d98937c..15dcc03f4 100644 --- a/frame.go +++ b/frame.go @@ -718,7 +718,7 @@ func (f *Frame) Value(columnID uint64, name string) (value int64, exists bool, e return 0, false, nil } - v, exists, err := view.FieldValue(columnID, field.BitDepth()) + v, exists, err := view.value(columnID, field.BitDepth()) if err != nil { return 0, false, err } else if !exists { @@ -764,7 +764,7 @@ func (f *Frame) Sum(filter *Row, name string) (sum, count int64, err error) { return 0, 0, nil } - vsum, vcount, err := view.FieldSum(filter, field.BitDepth()) + vsum, vcount, err := view.sum(filter, field.BitDepth()) if err != nil { return 0, 0, err } @@ -784,7 +784,7 @@ func (f *Frame) Min(filter *Row, name string) (min, count int64, err error) { return 0, 0, nil } - vmin, vcount, err := view.FieldMin(filter, bsig.BitDepth()) + vmin, vcount, err := view.min(filter, bsig.BitDepth()) if err != nil { return 0, 0, err } @@ -804,7 +804,7 @@ func (f *Frame) Max(filter *Row, name string) (max, count int64, err error) { return 0, 0, nil } - vmax, vcount, err := view.FieldMax(filter, field.BitDepth()) + vmax, vcount, err := view.max(filter, field.BitDepth()) if err != nil { return 0, 0, err } @@ -831,7 +831,7 @@ func (f *Frame) Range(name string, op pql.Token, predicate int64) (*Row, error) return NewRow(), nil } - return view.FieldRange(op, field.BitDepth(), baseValue) + return view.rangeOp(op, field.BitDepth(), baseValue) } func (f *Frame) RangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { @@ -854,7 +854,7 @@ func (f *Frame) RangeBetween(name string, predicateMin, predicateMax int64) (*Ro return NewRow(), nil } - return view.FieldRangeBetween(field.BitDepth(), baseValueMin, baseValueMax) + return view.rangeBetween(field.BitDepth(), baseValueMin, baseValueMax) } // Import bulk imports data. diff --git a/view.go b/view.go index 994a54579..31bf9a470 100644 --- a/view.go +++ b/view.go @@ -323,8 +323,8 @@ func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { return frag.ClearBit(rowID, columnID) } -// FieldValue uses a column of bits to read a multi-bit value. -func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +// value uses a column of bits to read a multi-bit value. +func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { @@ -343,8 +343,8 @@ func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed b return frag.SetValue(columnID, bitDepth, value) } -// FieldSum returns the sum & count of a field. -func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) { +// sum returns the sum & count of a field. +func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { for _, f := range v.Fragments() { fsum, fcount, err := f.FieldSum(filter, bitDepth) if err != nil { @@ -356,8 +356,8 @@ func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err erro return sum, count, nil } -// FieldMin returns the min and count of a field. -func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) { +// min returns the min and count of a field. +func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) { var minHasValue bool for _, f := range v.Fragments() { fmin, fcount, err := f.FieldMin(filter, bitDepth) @@ -384,8 +384,8 @@ func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err erro return min, count, nil } -// FieldMax returns the max and count of a field. -func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) { +// max returns the max and count of a field. +func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) { for _, f := range v.Fragments() { fmax, fcount, err := f.FieldMax(filter, bitDepth) if err != nil { @@ -399,8 +399,8 @@ func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err erro return max, count, nil } -// FieldRange returns rows with a field value encoding matching the predicate. -func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +// rangeOp returns rows with a field value encoding matching the predicate. +func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { r := NewRow() for _, frag := range v.Fragments() { other, err := frag.FieldRange(op, bitDepth, predicate) @@ -412,9 +412,9 @@ func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, return r, nil } -// FieldRangeBetween returns bitmaps with a field value encoding matching any +// rangeBetween returns bitmaps with a field value encoding matching any // value between predicateMin and predicateMax. -func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { +func (v *View) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { r := NewRow() for _, frag := range v.Fragments() { other, err := frag.FieldRangeBetween(bitDepth, predicateMin, predicateMax) From 51a42d3e359b5dc03102d83c25c20252f736f0a9 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Jun 2018 17:40:25 -0500 Subject: [PATCH 22/78] rename test names --- frame_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frame_test.go b/frame_test.go index aeae2b7ba..d3e274fe9 100644 --- a/frame_test.go +++ b/frame_test.go @@ -147,7 +147,7 @@ func TestFrame_SetValue(t *testing.T) { } }) - t.Run("ErrFieldNotFound", func(t *testing.T) { + t.Run("ErrBSIGroupNotFound", func(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() @@ -164,7 +164,7 @@ func TestFrame_SetValue(t *testing.T) { } }) - t.Run("ErrFieldValueTooLow", func(t *testing.T) { + t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() @@ -183,7 +183,7 @@ func TestFrame_SetValue(t *testing.T) { } }) - t.Run("ErrFieldValueTooHigh", func(t *testing.T) { + t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() From b566fd278a4399229954426fc421a3bc578efdfc Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Jun 2018 20:00:30 -0500 Subject: [PATCH 23/78] remove field name argument from Frame.Value() --- executor_test.go | 4 ++-- frame.go | 6 +++--- frame_test.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/executor_test.go b/executor_test.go index 4b75686a4..095e45bf4 100644 --- a/executor_test.go +++ b/executor_test.go @@ -290,7 +290,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } f := hldr.Frame("i", "f") - if value, exists, err := f.Value(10, "f"); err != nil { + if value, exists, err := f.Value(10); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") @@ -298,7 +298,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Fatalf("unexpected value: %v", value) } - if value, exists, err := f.Value(100, "f"); err != nil { + if value, exists, err := f.Value(100); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") diff --git a/frame.go b/frame.go index 15dcc03f4..4d5e70510 100644 --- a/frame.go +++ b/frame.go @@ -706,14 +706,14 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change } // Value reads a bsiGroup value for a column. -func (f *Frame) Value(columnID uint64, name string) (value int64, exists bool, err error) { - field := f.bsiGroup(name) +func (f *Frame) Value(columnID uint64) (value int64, exists bool, err error) { + field := f.bsiGroup(f.name) if field == nil { return 0, false, ErrBSIGroupNotFound } // Fetch target view. - view := f.View(viewBSIGroupPrefix + name) + view := f.View(viewBSIGroupPrefix + f.name) if view == nil { return 0, false, nil } diff --git a/frame_test.go b/frame_test.go index d3e274fe9..739b34ed8 100644 --- a/frame_test.go +++ b/frame_test.go @@ -94,7 +94,7 @@ func TestFrame_SetValue(t *testing.T) { } // Read value. - if value, exists, err := f.Value(100, "f"); err != nil { + if value, exists, err := f.Value(100); err != nil { t.Fatal(err) } else if value != 21 { t.Fatalf("unexpected value: %d", value) @@ -138,7 +138,7 @@ func TestFrame_SetValue(t *testing.T) { } // Read value. - if value, exists, err := f.Value(100, "f"); err != nil { + if value, exists, err := f.Value(100); err != nil { t.Fatal(err) } else if value != 23 { t.Fatalf("unexpected value: %d", value) From 28cdaa61e725f85818e620dd615e55189cf5675b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Jun 2018 20:08:24 -0500 Subject: [PATCH 24/78] rename some instances of field to bsiGroup --- frame.go | 81 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/frame.go b/frame.go index 4d5e70510..2b9e03993 100644 --- a/frame.go +++ b/frame.go @@ -424,7 +424,7 @@ func (f *Frame) addBSIGroup(bsig *bsiGroup) error { // Add bsiGroup to list. f.bsiGroups = append(f.bsiGroups, bsig) - // Sort fields by name. + // Sort bsiGroups by name. sort.Slice(f.bsiGroups, func(i, j int) bool { return f.bsiGroups[i].Name < f.bsiGroups[j].Name }) @@ -432,13 +432,12 @@ func (f *Frame) addBSIGroup(bsig *bsiGroup) error { return nil } -// TODO: merge this into the un-exported deleteBSIGroup. -// deleteBSIGroupAndView deletes an existing field on the schema. +// deleteBSIGroupAndView deletes an existing bsiGroup on the schema. func (f *Frame) deleteBSIGroupAndView(name string) error { f.mu.Lock() defer f.mu.Unlock() - // Remove field. + // Remove bsiGroup. if err := f.deleteBSIGroup(name); err != nil { return err } @@ -705,10 +704,10 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change return changed, nil } -// Value reads a bsiGroup value for a column. +// Value reads a frame value for a column. func (f *Frame) Value(columnID uint64) (value int64, exists bool, err error) { - field := f.bsiGroup(f.name) - if field == nil { + bsig := f.bsiGroup(f.name) + if bsig == nil { return 0, false, ErrBSIGroupNotFound } @@ -718,24 +717,24 @@ func (f *Frame) Value(columnID uint64) (value int64, exists bool, err error) { return 0, false, nil } - v, exists, err := view.value(columnID, field.BitDepth()) + v, exists, err := view.value(columnID, bsig.BitDepth()) if err != nil { return 0, false, err } else if !exists { return 0, false, nil } - return int64(v) + field.Min, true, nil + return int64(v) + bsig.Min, true, nil } -// SetValue sets a field value for a column. +// SetValue sets a frame value for a column. func (f *Frame) SetValue(columnID uint64, value int64) (changed bool, err error) { - // Fetch field and validate value. - field := f.bsiGroup(f.name) - if field == nil { + // Fetch bsiGroup and validate value. + bsig := f.bsiGroup(f.name) + if bsig == nil { return false, ErrBSIGroupNotFound - } else if value < field.Min { + } else if value < bsig.Min { return false, ErrBSIGroupValueTooLow - } else if value > field.Max { + } else if value > bsig.Max { return false, ErrBSIGroupValueTooHigh } @@ -746,16 +745,16 @@ func (f *Frame) SetValue(columnID uint64, value int64) (changed bool, err error) } // Determine base value to store. - baseValue := uint64(value - field.Min) + baseValue := uint64(value - bsig.Min) - return view.setValue(columnID, field.BitDepth(), baseValue) + return view.setValue(columnID, bsig.BitDepth(), baseValue) } -// Sum returns the sum and count for a field. +// Sum returns the sum and count for a frame. // An optional filtering row can be provided. func (f *Frame) Sum(filter *Row, name string) (sum, count int64, err error) { - field := f.bsiGroup(name) - if field == nil { + bsig := f.bsiGroup(name) + if bsig == nil { return 0, 0, ErrBSIGroupNotFound } @@ -764,14 +763,14 @@ func (f *Frame) Sum(filter *Row, name string) (sum, count int64, err error) { return 0, 0, nil } - vsum, vcount, err := view.sum(filter, field.BitDepth()) + vsum, vcount, err := view.sum(filter, bsig.BitDepth()) if err != nil { return 0, 0, err } - return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil + return int64(vsum) + (int64(vcount) * bsig.Min), int64(vcount), nil } -// Min returns the min for a field. +// Min returns the min for a frame. // An optional filtering row can be provided. func (f *Frame) Min(filter *Row, name string) (min, count int64, err error) { bsig := f.bsiGroup(name) @@ -791,11 +790,11 @@ func (f *Frame) Min(filter *Row, name string) (min, count int64, err error) { return int64(vmin) + bsig.Min, int64(vcount), nil } -// Max returns the max for a field. +// Max returns the max for a frame. // An optional filtering row can be provided. func (f *Frame) Max(filter *Row, name string) (max, count int64, err error) { - field := f.bsiGroup(name) - if field == nil { + bsig := f.bsiGroup(name) + if bsig == nil { return 0, 0, ErrBSIGroupNotFound } @@ -804,57 +803,57 @@ func (f *Frame) Max(filter *Row, name string) (max, count int64, err error) { return 0, 0, nil } - vmax, vcount, err := view.max(filter, field.BitDepth()) + vmax, vcount, err := view.max(filter, bsig.BitDepth()) if err != nil { return 0, 0, err } - return int64(vmax) + field.Min, int64(vcount), nil + return int64(vmax) + bsig.Min, int64(vcount), nil } func (f *Frame) Range(name string, op pql.Token, predicate int64) (*Row, error) { - // Retrieve and validate field. - field := f.bsiGroup(name) - if field == nil { + // Retrieve and validate bsiGroup. + bsig := f.bsiGroup(name) + if bsig == nil { return nil, ErrBSIGroupNotFound - } else if predicate < field.Min || predicate > field.Max { + } else if predicate < bsig.Min || predicate > bsig.Max { return nil, nil } - // Retrieve field's view. + // Retrieve bsiGroup's view. view := f.View(viewBSIGroupPrefix + name) if view == nil { return nil, nil } - baseValue, outOfRange := field.baseValue(op, predicate) + baseValue, outOfRange := bsig.baseValue(op, predicate) if outOfRange { return NewRow(), nil } - return view.rangeOp(op, field.BitDepth(), baseValue) + return view.rangeOp(op, bsig.BitDepth(), baseValue) } func (f *Frame) RangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { - // Retrieve and validate field. - field := f.bsiGroup(name) - if field == nil { + // Retrieve and validate bsiGroup. + bsig := f.bsiGroup(name) + if bsig == nil { return nil, ErrBSIGroupNotFound } else if predicateMin > predicateMax { return nil, ErrInvalidBetweenValue } - // Retrieve field's view. + // Retrieve bsiGroup's view. view := f.View(viewBSIGroupPrefix + name) if view == nil { return nil, nil } - baseValueMin, baseValueMax, outOfRange := field.baseValueBetween(predicateMin, predicateMax) + baseValueMin, baseValueMax, outOfRange := bsig.baseValueBetween(predicateMin, predicateMax) if outOfRange { return NewRow(), nil } - return view.rangeBetween(field.BitDepth(), baseValueMin, baseValueMax) + return view.rangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) } // Import bulk imports data. From 2111a3d5219d18db17d0a778d6411888d5f4d051 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 11:13:51 -0500 Subject: [PATCH 25/78] more Field removal/rename --- api.go | 4 +- client.go | 9 ++- client_test.go | 2 +- ctl/import.go | 2 +- executor.go | 34 +++++----- executor_test.go | 16 ++--- fragment.go | 6 +- fragment_test.go | 2 +- frame.go | 12 ++-- frame_internal_test.go | 4 +- internal/public.pb.go | 138 ++++++++++++++--------------------------- internal/public.proto | 1 - 12 files changed, 90 insertions(+), 140 deletions(-) diff --git a/api.go b/api.go index e34a9142b..6f6123f9b 100644 --- a/api.go +++ b/api.go @@ -710,9 +710,9 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest } // Import into fragment. - err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values) + err = frame.ImportValue(req.ColumnIDs, req.Values) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, columns=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } diff --git a/client.go b/client.go index f3e0e5dce..1d78dde23 100644 --- a/client.go +++ b/client.go @@ -420,14 +420,14 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []b } // ImportValue bulk imports field values for a single slice to a host. -func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error { +func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame string, slice uint64, vals []FieldValue) error { if index == "" { return ErrIndexRequired } else if frame == "" { return ErrFrameRequired } - buf, err := marshalImportValuePayload(index, frame, field, slice, vals) + buf, err := marshalImportValuePayload(index, frame, slice, vals) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -449,7 +449,7 @@ func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, fiel } // marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func marshalImportValuePayload(index, frame, field string, slice uint64, vals []FieldValue) ([]byte, error) { +func marshalImportValuePayload(index, frame string, slice uint64, vals []FieldValue) ([]byte, error) { // Separate row and column IDs to reduce allocations. columnIDs := FieldValues(vals).ColumnIDs() values := FieldValues(vals).Values() @@ -459,7 +459,6 @@ func marshalImportValuePayload(index, frame, field string, slice uint64, vals [] Index: index, Frame: frame, Slice: slice, - Field: field, ColumnIDs: columnIDs, Values: values, }) @@ -1058,7 +1057,7 @@ type InternalClient interface { ImportK(ctx context.Context, index, frame string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error - ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error + ImportValue(ctx context.Context, index, frame string, slice uint64, vals []FieldValue) error ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) diff --git a/client_test.go b/client_test.go index 2bb255382..1edd37bc2 100644 --- a/client_test.go +++ b/client_test.go @@ -267,7 +267,7 @@ func TestClient_ImportValue(t *testing.T) { // Send import request. c := test.MustNewClient(s.Host(), defaultClient) - if err := c.ImportValue(context.Background(), "i", "f", fldName, 0, []pilosa.FieldValue{ + if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{ {ColumnID: 1, Value: -10}, {ColumnID: 2, Value: 20}, {ColumnID: 3, Value: 40}, diff --git a/ctl/import.go b/ctl/import.go index a9d70fd7c..4fdfba444 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -435,7 +435,7 @@ func (cmd *ImportCommand) importFieldValues(ctx context.Context, vals []pilosa.F } logger.Printf("importing slice: %d, n=%d", slice, len(vals)) - if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, cmd.Field, slice, vals); err != nil { + if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, slice, vals); err != nil { return errors.Wrap(err, "importing values") } } diff --git a/executor.go b/executor.go index b5e19e5b7..e6c8ad251 100644 --- a/executor.go +++ b/executor.go @@ -127,10 +127,10 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeSum(ctx, index, c, slices, opt) case "Min": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeFieldMin(ctx, index, c, slices, opt) + return e.executeMin(ctx, index, c, slices, opt) case "Max": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeFieldMax(ctx, index, c, slices, opt) + return e.executeMax(ctx, index, c, slices, opt) case "ClearBit": return e.executeClearBit(ctx, index, c, opt) case "Count": @@ -207,8 +207,8 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl return other, nil } -// executeFieldMin executes a Min() call. -func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { +// executeMin executes a Min() call. +func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame := c.Args["frame"]; frame == "" { return ValCount{}, errors.New("Min(): frame required") } else if field := c.Args["field"]; field == "" { @@ -221,7 +221,7 @@ func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Cal // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { - return e.executeFieldMinSlice(ctx, index, c, slice) + return e.executeMinSlice(ctx, index, c, slice) } // Merge returned results at coordinating node. @@ -242,8 +242,8 @@ func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Cal return other, nil } -// executeFieldMax executes a Max() call. -func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { +// executeMax executes a Max() call. +func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame := c.Args["frame"]; frame == "" { return ValCount{}, errors.New("Max(): frame required") } else if field := c.Args["field"]; field == "" { @@ -256,7 +256,7 @@ func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Cal // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { - return e.executeFieldMaxSlice(ctx, index, c, slice) + return e.executeMaxSlice(ctx, index, c, slice) } // Merge returned results at coordinating node. @@ -401,8 +401,8 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq }, nil } -// executeFieldMinSlice calculates the min for fields on a slice. -func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeMinSlice calculates the min for fields on a slice. +func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) @@ -440,8 +440,8 @@ func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pq }, nil } -// executeFieldMaxSlice calculates the max for fields on a slice. -func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeMaxSlice calculates the max for fields on a slice. +func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) @@ -552,7 +552,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) } - field, _ := c.Args["field"].(string) + field, _ := c.Args["field"].(string) // TODO: rename this to something other than field rowIDs, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) @@ -600,7 +600,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca N: int(n), Src: src, RowIDs: rowIDs, - FilterField: field, + FilterName: field, FilterValues: filters, MinThreshold: minThreshold, TanimotoThreshold: tanimotoThreshold, @@ -687,7 +687,7 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { // Handle field ranges differently. if c.HasConditionArg() { - return e.executeFieldRangeSlice(ctx, index, c, slice) + return e.executeBSIGroupRangeSlice(ctx, index, c, slice) } // Parse frame, use default if unset. @@ -756,8 +756,8 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return row, nil } -// executeFieldRangeSlice executes a range(field) call for a local slice. -func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeBSIGroupRangeSlice executes a range(field) call for a local slice. +func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { // Parse frame, use default if unset. frame, _ := c.Args["frame"].(string) if frame == "" { diff --git a/executor_test.go b/executor_test.go index 095e45bf4..606646a92 100644 --- a/executor_test.go +++ b/executor_test.go @@ -319,21 +319,21 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Fatal(err) } - t.Run("ErrColumnFieldRequired", func(t *testing.T) { + t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrColumnFieldValue", func(t *testing.T) { + t.Run("ErrColumnBSIGroupValue", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrInvalidFieldValueType", func(t *testing.T) { + t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err != pilosa.ErrInvalidBSIGroupValueType { t.Fatalf("unexpected error: %s", err) @@ -728,7 +728,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { } // Ensure a range query can be executed. -func TestExecutor_Execute_Range(t *testing.T) { +func TestExecutor_Execute_BSIGroupRange(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) @@ -770,7 +770,7 @@ func TestExecutor_Execute_Range(t *testing.T) { } // Ensure a Range(field) query can be executed. -func TestExecutor_Execute_FieldRange(t *testing.T) { +func TestExecutor_Execute_Range(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) @@ -903,8 +903,8 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } }) - // Ensure that the FieldNotNull code path gets run. - t.Run("FieldNotNull", func(t *testing.T) { + // Ensure that the NotNull code path gets run. + t.Run("NotNull", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, other >< [0, 1000])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { @@ -950,7 +950,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } }) - t.Run("ErrFieldNotFound", func(t *testing.T) { + t.Run("ErrBSIGroupNotFound", func(t *testing.T) { if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, bad_field >= 20)`), nil, nil); err != pilosa.ErrBSIGroupNotFound { t.Fatal(err) } diff --git a/fragment.go b/fragment.go index f2d7ad986..fe50a76f2 100644 --- a/fragment.go +++ b/fragment.go @@ -900,7 +900,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Create a fast lookup of filter values. var filters map[interface{}]struct{} - if opt.FilterField != "" && len(opt.FilterValues) > 0 { + if opt.FilterName != "" && len(opt.FilterValues) > 0 { filters = make(map[interface{}]struct{}) for _, v := range opt.FilterValues { filters[v] = struct{}{} @@ -948,7 +948,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { return nil, errors.Wrap(err, "getting attrs") } else if attr == nil { continue - } else if attrValue := attr[opt.FilterField]; attrValue == nil { + } else if attrValue := attr[opt.FilterName]; attrValue == nil { continue } else if _, ok := filters[attrValue]; !ok { continue @@ -1074,7 +1074,7 @@ type TopOptions struct { MinThreshold uint64 // Filter field name & values. - FilterField string + FilterName string FilterValues []interface{} TanimotoThreshold uint64 } diff --git a/fragment_test.go b/fragment_test.go index f5205eb73..6655f62b9 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -623,7 +623,7 @@ func TestFragment_Top_Filter(t *testing.T) { // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{ N: 2, - FilterField: "x", + FilterName: "x", FilterValues: []interface{}{int64(10), int64(15), int64(20)}, }); err != nil { t.Fatal(err) diff --git a/frame.go b/frame.go index 2b9e03993..a34eabf7f 100644 --- a/frame.go +++ b/frame.go @@ -914,12 +914,12 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro } // ImportValue bulk imports range-encoded value data. -func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { - viewName := viewBSIGroupPrefix + fieldName +func (f *Frame) ImportValue(columnIDs []uint64, values []int64) error { + viewName := viewBSIGroupPrefix + f.name // Get the bsiGroup so we know bitDepth. - bsig := f.bsiGroup(fieldName) + bsig := f.bsiGroup(f.name) if bsig == nil { - return errors.Wrap(ErrBSIGroupNotFound, fieldName) + return errors.Wrap(ErrBSIGroupNotFound, f.name) } // Split import data by fragment. @@ -1084,7 +1084,7 @@ func isValidBSIGroupType(v string) bool { } } -// bsiGroup represents a range field on a frame. +// bsiGroup represents a group of range-encoded rows on a frame. type bsiGroup struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` @@ -1113,7 +1113,7 @@ func (b *bsiGroup) BitDepth() uint { // we can't simply return 1024. // In order to make this work, we effectively need to change the operator to LTE. // Executor.executeFieldRangeSlice() takes this into account and returns -// `frag.FieldNotNull(field.BitDepth())` in such instances. +// `frag.FieldNotNull(bsig.BitDepth())` in such instances. func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { if op == pql.GT || op == pql.GTE { if value > b.Max { diff --git a/frame_internal_test.go b/frame_internal_test.go index 5cba2e84b..91b1af02c 100644 --- a/frame_internal_test.go +++ b/frame_internal_test.go @@ -21,8 +21,8 @@ import ( "github.com/pilosa/pilosa/pql" ) -// Ensure a field can adjust to its baseValue. -func TestField_BaseValue(t *testing.T) { +// Ensure a bsiGroup can adjust to its baseValue. +func TestBSIGroup_BaseValue(t *testing.T) { b0 := &bsiGroup{ Name: "b0", Type: bsiGroupTypeInt, diff --git a/internal/public.pb.go b/internal/public.pb.go index 069f633c0..2fc4b3827 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -485,7 +485,6 @@ type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` - Field string `protobuf:"bytes,4,opt,name=Field,proto3" json:"Field,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` @@ -517,13 +516,6 @@ func (m *ImportValueRequest) GetSlice() uint64 { return 0 } -func (m *ImportValueRequest) GetField() string { - if m != nil { - return m.Field - } - return "" -} - func (m *ImportValueRequest) GetColumnIDs() []uint64 { if m != nil { return m.ColumnIDs @@ -1190,12 +1182,6 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(m.Slice)) } - if len(m.Field) > 0 { - dAtA[i] = 0x22 - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) - } if len(m.ColumnIDs) > 0 { dAtA14 := make([]byte, len(m.ColumnIDs)*10) var j13 int @@ -1545,10 +1531,6 @@ func (m *ImportValueRequest) Size() (n int) { if m.Slice != 0 { n += 1 + sovPublic(uint64(m.Slice)) } - l = len(m.Field) - if l > 0 { - n += 1 + l + sovPublic(uint64(l)) - } if len(m.ColumnIDs) > 0 { l = 0 for _, e := range m.ColumnIDs { @@ -3507,35 +3489,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { break } } - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPublic - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Field = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 5: if wireType == 0 { var v uint64 @@ -3818,50 +3771,49 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 709 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c, - 0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0xe8, 0xfb, 0x8a, 0x85, 0x50, 0xb0, 0x2c, - 0x84, 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0x60, 0x5a, 0x8a, 0x58, 0xba, - 0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x0c, 0xc4, - 0xaa, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x7b, 0xc7, 0x13, 0xbb, 0xa9, 0x04, 0x2c, 0xd8, 0xcd, - 0x39, 0x67, 0xe6, 0x66, 0xce, 0xdc, 0x73, 0x1d, 0xd8, 0xc8, 0xca, 0xb3, 0x24, 0x3e, 0x9f, 0x64, - 0xb9, 0xd2, 0x8a, 0xf7, 0xe2, 0x54, 0xcb, 0x3c, 0x8d, 0x92, 0xe0, 0x2d, 0x38, 0x42, 0x2d, 0xb8, - 0x07, 0xdd, 0x5d, 0x95, 0x94, 0xf3, 0xb4, 0xf0, 0x98, 0xef, 0x84, 0xae, 0xb0, 0x90, 0x3f, 0x82, - 0xf6, 0x73, 0xad, 0xf3, 0xc2, 0x6b, 0xf9, 0x4e, 0x38, 0x98, 0x8e, 0x26, 0xf6, 0xe8, 0x04, 0x69, - 0x61, 0x44, 0xce, 0xc1, 0x3d, 0x94, 0xcb, 0xc2, 0x73, 0x7c, 0x27, 0xec, 0x0b, 0x5a, 0x07, 0x4f, - 0xc1, 0x7d, 0x19, 0xc5, 0x39, 0x1f, 0x41, 0x6b, 0xb6, 0xe7, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd, - 0xf6, 0xf8, 0x7f, 0xd0, 0xde, 0x55, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x4d, 0x70, 0x0e, - 0xe5, 0xd2, 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0x4e, 0xa3, 0x64, 0xa5, 0x9e, - 0x46, 0x09, 0x15, 0x71, 0x04, 0x2e, 0x6f, 0x57, 0x71, 0xaa, 0x2a, 0xc1, 0x6b, 0x70, 0x76, 0x62, - 0x8d, 0xa2, 0x50, 0x8b, 0xd5, 0xaf, 0x1a, 0xc0, 0xef, 0x43, 0xcf, 0xb8, 0x9a, 0xed, 0x55, 0xbf, - 0xbd, 0xc2, 0xfc, 0x01, 0xf4, 0x4f, 0xe2, 0xb9, 0x2c, 0x74, 0x34, 0xcf, 0xe8, 0x12, 0x8e, 0xa8, - 0x89, 0xe0, 0x0d, 0x0c, 0xcd, 0x4e, 0x74, 0x7b, 0x2c, 0xf5, 0x1d, 0x4f, 0x7f, 0xf6, 0x4a, 0x77, - 0x3d, 0x7e, 0x62, 0xe0, 0xa2, 0x66, 0x25, 0xb6, 0x92, 0xf0, 0x49, 0x4f, 0x96, 0x99, 0xac, 0x6e, - 0x4a, 0x6b, 0xee, 0xc3, 0xe0, 0x58, 0xe7, 0x71, 0x7a, 0x79, 0x1a, 0x25, 0xa5, 0xac, 0x0a, 0x35, - 0x29, 0xf4, 0x38, 0x4b, 0xb5, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xe8, 0x71, 0x47, 0xa9, 0xc4, 0x88, - 0x6d, 0x9f, 0x85, 0x3d, 0x51, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xa8, 0xa8, 0x3a, 0xdb, 0xf1, 0x59, - 0xc8, 0x44, 0x83, 0x09, 0xb6, 0xa1, 0x8b, 0x37, 0x7d, 0x11, 0x65, 0xb5, 0x5b, 0xf6, 0x0b, 0xb7, - 0xc1, 0x35, 0x83, 0x8d, 0x57, 0xa5, 0xcc, 0x97, 0x42, 0xbe, 0x2f, 0x65, 0x41, 0x5d, 0x21, 0x5c, - 0xb9, 0x34, 0x80, 0x6f, 0x41, 0xe7, 0x38, 0x89, 0xcf, 0xa5, 0x79, 0x3b, 0x57, 0x54, 0x08, 0xbd, - 0xd6, 0x6f, 0x5e, 0x90, 0xd7, 0x9e, 0x68, 0x52, 0x78, 0x52, 0xc8, 0xb9, 0xd2, 0xd6, 0x4c, 0x85, - 0x78, 0x08, 0xff, 0xee, 0x5f, 0x9d, 0x27, 0xe5, 0x85, 0x14, 0x6a, 0x61, 0x4e, 0x77, 0x68, 0xc3, - 0x3a, 0xcd, 0x1f, 0xc3, 0xa8, 0xa2, 0x6c, 0xfa, 0xbb, 0xb4, 0x71, 0x8d, 0x0d, 0x3e, 0x30, 0x18, - 0x56, 0x56, 0x8a, 0x4c, 0xa5, 0x85, 0xc4, 0x7e, 0xed, 0xe7, 0xb9, 0xed, 0xd7, 0x7e, 0x9e, 0xf3, - 0x6d, 0xe8, 0x0a, 0x59, 0x94, 0x89, 0xb6, 0x21, 0xf8, 0xbf, 0x7e, 0x16, 0x7b, 0xb6, 0x4c, 0xb4, - 0xb0, 0xbb, 0xf8, 0x33, 0x18, 0xdd, 0x0a, 0x95, 0x99, 0x9e, 0xc1, 0xf4, 0x5e, 0x7d, 0xee, 0x96, - 0x2e, 0xd6, 0xb6, 0x07, 0x9f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x43, 0x9a, 0x65, 0xba, 0xd3, 0x60, - 0x3a, 0xac, 0xab, 0x08, 0xb5, 0x10, 0x34, 0xe5, 0x1b, 0xc0, 0x8e, 0xaa, 0x3c, 0xb1, 0x23, 0xec, - 0x22, 0xce, 0xa7, 0xfd, 0xd9, 0x46, 0x17, 0x91, 0x16, 0x46, 0xa4, 0x2f, 0xc3, 0xbb, 0x28, 0xbd, - 0x94, 0x17, 0x94, 0xa7, 0x9e, 0xb0, 0x90, 0x4f, 0xea, 0xf9, 0xa4, 0x06, 0x0c, 0xa6, 0xbc, 0x2e, - 0x61, 0x15, 0x51, 0xcf, 0xb0, 0x0d, 0x34, 0xf6, 0x62, 0x68, 0x02, 0x1d, 0x7c, 0x67, 0x30, 0x9c, - 0xcd, 0x33, 0x95, 0xeb, 0x46, 0x48, 0x66, 0xe9, 0x85, 0xbc, 0xb2, 0x21, 0x21, 0x80, 0xec, 0x41, - 0x1e, 0xcd, 0xcd, 0x34, 0xf4, 0x85, 0x01, 0xc8, 0x52, 0x58, 0x28, 0x1c, 0xae, 0x30, 0x80, 0x62, - 0x81, 0xf3, 0x5e, 0x78, 0xae, 0x09, 0x94, 0x41, 0x18, 0x7f, 0x3b, 0xee, 0x85, 0xd7, 0x26, 0xa9, - 0x26, 0x30, 0xfe, 0xab, 0x79, 0xc7, 0xbc, 0x38, 0xa1, 0x23, 0x1a, 0x0c, 0xbe, 0x83, 0x50, 0x0b, - 0xfa, 0xc8, 0x75, 0xe9, 0x23, 0x67, 0x21, 0x9e, 0x34, 0x65, 0x48, 0xec, 0x91, 0xd8, 0x60, 0x82, - 0x2f, 0x0c, 0xb8, 0xf1, 0x48, 0x83, 0xf4, 0xf7, 0x8c, 0xe2, 0xde, 0x58, 0x26, 0xa6, 0x31, 0xb8, - 0x17, 0xc1, 0x6f, 0x6c, 0x6e, 0x41, 0x87, 0x6e, 0x61, 0x2d, 0x56, 0x68, 0xcd, 0x44, 0x77, 0xdd, - 0xc4, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbd, 0x19, 0xb3, 0x6f, 0x37, 0x63, 0xf6, 0xf1, 0xc7, - 0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x95, 0x3c, 0xf9, 0x19, 0x00, 0x00, 0xff, 0xff, 0x03, 0x56, 0xc7, - 0xa4, 0x5a, 0x06, 0x00, 0x00, + // 701 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd4, 0x3c, + 0x14, 0xfd, 0x3c, 0xc9, 0xfc, 0xdd, 0xe9, 0xcc, 0x57, 0x59, 0xdf, 0x57, 0x22, 0x84, 0x86, 0x28, + 0x42, 0x28, 0xab, 0xa9, 0x34, 0xec, 0x41, 0xf4, 0x4f, 0x1a, 0x55, 0x54, 0x70, 0x5b, 0x8a, 0x58, + 0xa6, 0xad, 0x55, 0x22, 0x65, 0xe2, 0x90, 0x38, 0x9a, 0xce, 0x73, 0xb0, 0xe1, 0x11, 0x58, 0xf0, + 0x10, 0x2c, 0xbb, 0xe4, 0x11, 0xa0, 0xbc, 0x08, 0xf2, 0x75, 0x3c, 0x49, 0xa7, 0x52, 0xc5, 0x82, + 0x9d, 0xcf, 0x39, 0xf6, 0xb5, 0x8f, 0x7d, 0x6e, 0x02, 0x1b, 0x59, 0x79, 0x96, 0xc4, 0xe7, 0x93, + 0x2c, 0x97, 0x4a, 0xf2, 0x5e, 0x9c, 0x2a, 0x91, 0xa7, 0x51, 0x12, 0xbc, 0x07, 0x07, 0xe5, 0x82, + 0x7b, 0xd0, 0xdd, 0x95, 0x49, 0x39, 0x4f, 0x0b, 0x8f, 0xf9, 0x4e, 0xe8, 0xa2, 0x85, 0xfc, 0x09, + 0xb4, 0x5f, 0x2a, 0x95, 0x17, 0x5e, 0xcb, 0x77, 0xc2, 0xc1, 0x74, 0x34, 0xb1, 0x4b, 0x27, 0x9a, + 0x46, 0x23, 0x72, 0x0e, 0xee, 0xa1, 0x58, 0x16, 0x9e, 0xe3, 0x3b, 0x61, 0x1f, 0x69, 0x1c, 0x3c, + 0x07, 0xf7, 0x75, 0x14, 0xe7, 0x7c, 0x04, 0xad, 0xd9, 0x9e, 0xc7, 0x7c, 0x16, 0xba, 0xd8, 0x9a, + 0xed, 0xf1, 0xff, 0xa0, 0xbd, 0x2b, 0xcb, 0x54, 0x79, 0x2d, 0xa2, 0x0c, 0xe0, 0x9b, 0xe0, 0x1c, + 0x8a, 0xa5, 0xe7, 0xf8, 0x2c, 0xec, 0xa3, 0x1e, 0x06, 0x53, 0xe8, 0x9d, 0x46, 0xc9, 0x4a, 0x3d, + 0x8d, 0x12, 0x2a, 0xe2, 0xa0, 0x1e, 0xde, 0xae, 0xe2, 0x54, 0x55, 0x82, 0xb7, 0xe0, 0xec, 0xc4, + 0x4a, 0x8b, 0x28, 0x17, 0xab, 0x5d, 0x0d, 0xe0, 0x0f, 0xa1, 0x67, 0x5c, 0xcd, 0xf6, 0xaa, 0xbd, + 0x57, 0x98, 0x3f, 0x82, 0xfe, 0x49, 0x3c, 0x17, 0x85, 0x8a, 0xe6, 0x19, 0x1d, 0xc2, 0xc1, 0x9a, + 0x08, 0xde, 0xc1, 0xd0, 0xcc, 0xd4, 0x6e, 0x8f, 0x85, 0xba, 0xe3, 0xe9, 0xcf, 0x6e, 0xe9, 0xae, + 0xc7, 0x2f, 0x0c, 0x5c, 0xad, 0x59, 0x89, 0xad, 0x24, 0x7d, 0xa5, 0x27, 0xcb, 0x4c, 0x54, 0x27, + 0xa5, 0x31, 0xf7, 0x61, 0x70, 0xac, 0xf2, 0x38, 0xbd, 0x3c, 0x8d, 0x92, 0x52, 0x54, 0x85, 0x9a, + 0x94, 0xf6, 0x38, 0x4b, 0x95, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xda, 0xe3, 0x8e, 0x94, 0x89, 0x11, + 0xdb, 0x3e, 0x0b, 0x7b, 0x58, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xc8, 0xa8, 0x5a, 0xdb, 0xf1, 0x59, + 0xc8, 0xb0, 0xc1, 0x04, 0xdb, 0xd0, 0xd5, 0x27, 0x7d, 0x15, 0x65, 0xb5, 0x5b, 0x76, 0x8f, 0xdb, + 0xe0, 0x9a, 0xc1, 0xc6, 0x9b, 0x52, 0xe4, 0x4b, 0x14, 0x1f, 0x4b, 0x51, 0xd0, 0xab, 0x10, 0xae, + 0x5c, 0x1a, 0xc0, 0xb7, 0xa0, 0x73, 0x9c, 0xc4, 0xe7, 0xc2, 0xdc, 0x9d, 0x8b, 0x15, 0xd2, 0x5e, + 0xeb, 0x3b, 0x2f, 0xc8, 0x6b, 0x0f, 0x9b, 0x94, 0x5e, 0x89, 0x62, 0x2e, 0x95, 0x35, 0x53, 0x21, + 0x1e, 0xc2, 0xbf, 0xfb, 0x57, 0xe7, 0x49, 0x79, 0x21, 0x50, 0x2e, 0xcc, 0xea, 0x0e, 0x4d, 0x58, + 0xa7, 0xf9, 0x53, 0x18, 0x55, 0x94, 0x4d, 0x7f, 0x97, 0x26, 0xae, 0xb1, 0xc1, 0x27, 0x06, 0xc3, + 0xca, 0x4a, 0x91, 0xc9, 0xb4, 0x10, 0xfa, 0xbd, 0xf6, 0xf3, 0xdc, 0xbe, 0xd7, 0x7e, 0x9e, 0xf3, + 0x6d, 0xe8, 0xa2, 0x28, 0xca, 0x44, 0xd9, 0x10, 0xfc, 0x5f, 0x5f, 0x8b, 0x5d, 0x5b, 0x26, 0x0a, + 0xed, 0x2c, 0xfe, 0x02, 0x46, 0xb7, 0x42, 0x65, 0xba, 0x67, 0x30, 0x7d, 0x50, 0xaf, 0xbb, 0xa5, + 0xe3, 0xda, 0xf4, 0xe0, 0x1b, 0x83, 0x41, 0xa3, 0x32, 0x7f, 0x4c, 0xbd, 0x4c, 0x67, 0x1a, 0x4c, + 0x87, 0x75, 0x15, 0x94, 0x0b, 0xa4, 0x2e, 0xdf, 0x00, 0x76, 0x54, 0xe5, 0x89, 0x1d, 0xe9, 0x57, + 0xd4, 0xfd, 0x69, 0xb7, 0x6d, 0xbc, 0xa2, 0xa6, 0xd1, 0x88, 0xf4, 0x65, 0xf8, 0x10, 0xa5, 0x97, + 0xe2, 0x82, 0xf2, 0xd4, 0x43, 0x0b, 0xf9, 0xa4, 0xee, 0x4f, 0x7a, 0x80, 0xc1, 0x94, 0xd7, 0x25, + 0xac, 0x82, 0x75, 0x0f, 0xdb, 0x40, 0xeb, 0xb7, 0x18, 0x9a, 0x40, 0x07, 0x3f, 0x19, 0x0c, 0x67, + 0xf3, 0x4c, 0xe6, 0xaa, 0x11, 0x92, 0x59, 0x7a, 0x21, 0xae, 0x6c, 0x48, 0x08, 0x68, 0xf6, 0x20, + 0x8f, 0xe6, 0xa6, 0x1b, 0xfa, 0x68, 0x80, 0x66, 0x29, 0x2c, 0x14, 0x0e, 0x17, 0x0d, 0xa0, 0x58, + 0xe8, 0x7e, 0x2f, 0x3c, 0xd7, 0x04, 0xca, 0x20, 0x1d, 0x7f, 0xdb, 0xee, 0x85, 0xd7, 0x26, 0xa9, + 0x26, 0x74, 0xfc, 0x57, 0xfd, 0xae, 0xf3, 0xe2, 0x84, 0x0e, 0x36, 0x18, 0x7d, 0x0f, 0x28, 0x17, + 0xf4, 0x91, 0xeb, 0xd2, 0x47, 0xce, 0x42, 0xbd, 0xd2, 0x94, 0x21, 0xb1, 0x47, 0x62, 0x83, 0x09, + 0xbe, 0x32, 0xe0, 0xc6, 0x23, 0x35, 0xd2, 0xdf, 0x33, 0x7a, 0xbf, 0xa1, 0x2d, 0xe8, 0xd0, 0x7e, + 0xd6, 0x4c, 0x85, 0xd6, 0x8e, 0xdb, 0x5d, 0x3f, 0xee, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbf, + 0x19, 0xb3, 0x1f, 0x37, 0x63, 0xf6, 0xf9, 0xd7, 0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x34, 0x9e, 0xfd, + 0x0e, 0x00, 0x00, 0xff, 0xff, 0xc8, 0x5d, 0x77, 0x8c, 0x44, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 9207d3a67..b37eea98c 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -83,7 +83,6 @@ message ImportValueRequest { string Index = 1; string Frame = 2; uint64 Slice = 3; - string Field = 4; repeated uint64 ColumnIDs = 5; repeated string ColumnKeys = 7; repeated int64 Values = 6; From 207e9c2674e658a421d4fde34613956dd7dbf463 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 12:30:59 -0500 Subject: [PATCH 26/78] minor fixes --- frame.go | 2 +- index_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frame.go b/frame.go index a34eabf7f..ac9eb5ee7 100644 --- a/frame.go +++ b/frame.go @@ -1112,7 +1112,7 @@ func (b *bsiGroup) BitDepth() uint { // Note that in this case (because the range uses the full BitDepth 0 to 1023), // we can't simply return 1024. // In order to make this work, we effectively need to change the operator to LTE. -// Executor.executeFieldRangeSlice() takes this into account and returns +// Executor.executeBSIGroupRangeSlice() takes this into account and returns // `frag.FieldNotNull(bsig.BitDepth())` in such instances. func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { if op == pql.GT || op == pql.GTE { diff --git a/index_test.go b/index_test.go index 3a4ee48bc..86d9043ce 100644 --- a/index_test.go +++ b/index_test.go @@ -158,7 +158,7 @@ func TestIndex_CreateFrame(t *testing.T) { } }) - t.Run("ErrInvalidFieldRange", func(t *testing.T) { + t.Run("ErrInvalidBSIGroupRange", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() @@ -166,7 +166,7 @@ func TestIndex_CreateFrame(t *testing.T) { Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, }, - }); err != pilosa.ErrInvalidFieldRange { + }); err != pilosa.ErrInvalidBSIGroupRange { t.Fatal(err) } }) From 7f1ac8fdcdbd400c8fba7db30a4962c4b3e4d869 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 12:57:15 -0500 Subject: [PATCH 27/78] remove Field* from fragment.go --- executor.go | 18 +++++++++--------- fragment.go | 46 +++++++++++++++++++++++----------------------- fragment_test.go | 44 ++++++++++++++++++++++---------------------- view.go | 12 ++++++------ 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/executor.go b/executor.go index e6c8ad251..98982606d 100644 --- a/executor.go +++ b/executor.go @@ -391,7 +391,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq return ValCount{}, nil } - vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth()) + vsum, vcount, err := fragment.Sum(filter, field.BitDepth()) if err != nil { return ValCount{}, errors.Wrap(err, "computing sum") } @@ -430,7 +430,7 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth()) + fmin, fcount, err := fragment.Min(filter, field.BitDepth()) if err != nil { return ValCount{}, err } @@ -469,7 +469,7 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth()) + fmax, fcount, err := fragment.Max(filter, field.BitDepth()) if err != nil { return ValCount{}, err } @@ -811,7 +811,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, return NewRow(), nil } - return frag.FieldNotNull(field.BitDepth()) + return frag.NotNull(field.BitDepth()) } else if cond.Op == pql.BETWEEN { @@ -849,10 +849,10 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, // If the query is asking for the entire valid range, just return // the not-null bitmap for the field. if predicates[0] <= field.Min && predicates[1] >= field.Max { - return frag.FieldNotNull(field.BitDepth()) + return frag.NotNull(field.BitDepth()) } - return frag.FieldRangeBetween(field.BitDepth(), baseValueMin, baseValueMax) + return frag.RangeBetween(field.BitDepth(), baseValueMin, baseValueMax) } else { @@ -882,16 +882,16 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid field range. if (cond.Op == pql.LT && value > field.Max) || (cond.Op == pql.LTE && value >= field.Max) || (cond.Op == pql.GT && value < field.Min) || (cond.Op == pql.GTE && value <= field.Min) { - return frag.FieldNotNull(field.BitDepth()) + return frag.NotNull(field.BitDepth()) } // outOfRange for NEQ should return all not-null. if outOfRange && cond.Op == pql.NEQ { - return frag.FieldNotNull(field.BitDepth()) + return frag.NotNull(field.BitDepth()) } f.Stats.Count("range:field", 1, 1.0) - return frag.FieldRange(cond.Op, field.BitDepth(), baseValue) + return frag.RangeOp(cond.Op, field.BitDepth(), baseValue) } } diff --git a/fragment.go b/fragment.go index fe50a76f2..dd0f05ed2 100644 --- a/fragment.go +++ b/fragment.go @@ -486,8 +486,8 @@ func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { return f.storage.Contains(pos), nil } -// FieldValue uses a column of bits to read a multi-bit value. -func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +// Value uses a column of bits to read a multi-bit value. +func (f *Fragment) Value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { f.mu.Lock() defer f.mu.Unlock() @@ -582,9 +582,9 @@ func (f *Fragment) importSetValue(columnID uint64, bitDepth uint, value uint64) return changed, nil } -// FieldSum returns the sum of a given field as well as the number of columns involved. +// Sum returns the sum of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) { +func (f *Fragment) Sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { // Compute count based on the existence row. row := f.Row(uint64(bitDepth)) if filter != nil { @@ -614,9 +614,9 @@ func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err return sum, count, nil } -// FieldMin returns the min of a given field as well as the number of columns involved. +// Min returns the min of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) { +func (f *Fragment) Min(filter *Row, bitDepth uint) (min, count uint64, err error) { consider := f.Row(uint64(bitDepth)) if filter != nil { @@ -647,9 +647,9 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err return min, count, nil } -// FieldMax returns the max of a given field as well as the number of columns involved. +// Max returns the max of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) { +func (f *Fragment) Max(filter *Row, bitDepth uint) (max, count uint64, err error) { consider := f.Row(uint64(bitDepth)) if filter != nil { @@ -678,23 +678,23 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err return max, count, nil } -// FieldRange returns bitmaps with a field value encoding matching the predicate. -func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +// RangeOp returns bitmaps with a bsiGroup value encoding matching the predicate. +func (f *Fragment) RangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { switch op { case pql.EQ: - return f.fieldRangeEQ(bitDepth, predicate) + return f.rangeEQ(bitDepth, predicate) case pql.NEQ: - return f.fieldRangeNEQ(bitDepth, predicate) + return f.rangeNEQ(bitDepth, predicate) case pql.LT, pql.LTE: - return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE) + return f.rangeLT(bitDepth, predicate, op == pql.LTE) case pql.GT, pql.GTE: - return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE) + return f.rangeGT(bitDepth, predicate, op == pql.GTE) default: return nil, ErrInvalidRangeOperation } } -func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *Fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. b := f.Row(uint64(bitDepth)) @@ -713,12 +713,12 @@ func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) { return b, nil } -func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *Fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. b := f.Row(uint64(bitDepth)) // Get the equal bitmap. - eq, err := f.fieldRangeEQ(bitDepth, predicate) + eq, err := f.rangeEQ(bitDepth, predicate) if err != nil { return nil, err } @@ -729,7 +729,7 @@ func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) return b, nil } -func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { +func (f *Fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { keep := NewRow() // Start with set of columns with values set. @@ -777,7 +777,7 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b return b, nil } -func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { +func (f *Fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { b := f.Row(uint64(bitDepth)) keep := NewRow() @@ -812,13 +812,13 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b return b, nil } -// FieldNotNull returns the not-null row (stored at bitDepth). -func (f *Fragment) FieldNotNull(bitDepth uint) (*Row, error) { +// NotNull returns the not-null row (stored at bitDepth). +func (f *Fragment) NotNull(bitDepth uint) (*Row, error) { return f.Row(uint64(bitDepth)), nil } -// FieldRangeBetween returns bitmaps with a field value encoding matching any value between predicateMin and predicateMax. -func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { +// RangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. +func (f *Fragment) RangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { b := f.Row(uint64(bitDepth)) keep1 := NewRow() // GTE keep2 := NewRow() // LTE diff --git a/fragment_test.go b/fragment_test.go index 6655f62b9..e1c9b7fd4 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -110,7 +110,7 @@ func TestFragment_SetValue(t *testing.T) { } // Read value. - if value, exists, err := f.FieldValue(100, 16); err != nil { + if value, exists, err := f.Value(100, 16); err != nil { t.Fatal(err) } else if value != 3829 { t.Fatalf("unexpected value: %d", value) @@ -145,7 +145,7 @@ func TestFragment_SetValue(t *testing.T) { } // Read value. - if value, exists, err := f.FieldValue(100, 16); err != nil { + if value, exists, err := f.Value(100, 16); err != nil { t.Fatal(err) } else if value != 2028 { t.Fatalf("unexpected value: %d", value) @@ -166,7 +166,7 @@ func TestFragment_SetValue(t *testing.T) { } // Non-existent value. - if value, exists, err := f.FieldValue(100, 11); err != nil { + if value, exists, err := f.Value(100, 11); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -202,7 +202,7 @@ func TestFragment_SetValue(t *testing.T) { // Ensure values are set. for columnID, value := range m { - v, exists, err := f.FieldValue(columnID, bitDepth) + v, exists, err := f.Value(columnID, bitDepth) if err != nil { t.Fatal(err) } else if value != int64(v) { @@ -238,7 +238,7 @@ func TestFragment_FieldSum(t *testing.T) { } t.Run("NoFilter", func(t *testing.T) { - if sum, n, err := f.FieldSum(nil, bitDepth); err != nil { + if sum, n, err := f.Sum(nil, bitDepth); err != nil { t.Fatal(err) } else if n != 4 { t.Fatalf("unexpected count: %d", n) @@ -248,7 +248,7 @@ func TestFragment_FieldSum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if sum, n, err := f.FieldSum(pilosa.NewRow(2000, 4000, 5000), bitDepth); err != nil { + if sum, n, err := f.Sum(pilosa.NewRow(2000, 4000, 5000), bitDepth); err != nil { t.Fatal(err) } else if n != 2 { t.Fatalf("unexpected count: %d", n) @@ -296,7 +296,7 @@ func TestFragment_FieldMinMax(t *testing.T) { {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil { + if min, cnt, err := f.Min(test.filter, bitDepth); err != nil { t.Fatal(err) } else if min != test.exp { t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) @@ -320,7 +320,7 @@ func TestFragment_FieldMinMax(t *testing.T) { {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil { + if max, cnt, err := f.Max(test.filter, bitDepth); err != nil { t.Fatal(err) } else if max != test.exp { t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) @@ -351,7 +351,7 @@ func TestFragment_FieldRange(t *testing.T) { } // Query for equality. - if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil { + if b, err := f.RangeOp(pql.EQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -374,7 +374,7 @@ func TestFragment_FieldRange(t *testing.T) { } // Query for inequality. - if b, err := f.FieldRange(pql.NEQ, bitDepth, 300); err != nil { + if b, err := f.RangeOp(pql.NEQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -401,28 +401,28 @@ func TestFragment_FieldRange(t *testing.T) { } // Query for fields less than (ending with set column). - if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil { + if b, err := f.RangeOp(pql.LT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields less than (ending with unset column). - if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil { + if b, err := f.RangeOp(pql.LT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields less than or equal to (ending with set column). - if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil { + if b, err := f.RangeOp(pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields less than or equal to (ending with unset column). - if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil { + if b, err := f.RangeOp(pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -449,28 +449,28 @@ func TestFragment_FieldRange(t *testing.T) { } // Query for fields greater than (ending with unset bit). - if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil { + if b, err := f.RangeOp(pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than (ending with set bit). - if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil { + if b, err := f.RangeOp(pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than or equal to (ending with unset bit). - if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil { + if b, err := f.RangeOp(pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than or equal to (ending with set bit). - if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil { + if b, err := f.RangeOp(pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -497,28 +497,28 @@ func TestFragment_FieldRange(t *testing.T) { } // Query for fields greater than (ending with unset column). - if b, err := f.FieldRangeBetween(bitDepth, 300, 2817); err != nil { + if b, err := f.RangeBetween(bitDepth, 300, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than (ending with set column). - if b, err := f.FieldRangeBetween(bitDepth, 301, 2817); err != nil { + if b, err := f.RangeBetween(bitDepth, 301, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than or equal to (ending with unset column). - if b, err := f.FieldRangeBetween(bitDepth, 301, 2816); err != nil { + if b, err := f.RangeBetween(bitDepth, 301, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than or equal to (ending with set column). - if b, err := f.FieldRangeBetween(bitDepth, 300, 2816); err != nil { + if b, err := f.RangeBetween(bitDepth, 300, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) diff --git a/view.go b/view.go index 31bf9a470..38c76a162 100644 --- a/view.go +++ b/view.go @@ -330,7 +330,7 @@ func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool, if err != nil { return value, exists, err } - return frag.FieldValue(columnID, bitDepth) + return frag.Value(columnID, bitDepth) } // setValue uses a column of bits to set a multi-bit value. @@ -346,7 +346,7 @@ func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed b // sum returns the sum & count of a field. func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { for _, f := range v.Fragments() { - fsum, fcount, err := f.FieldSum(filter, bitDepth) + fsum, fcount, err := f.Sum(filter, bitDepth) if err != nil { return sum, count, err } @@ -360,7 +360,7 @@ func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) { var minHasValue bool for _, f := range v.Fragments() { - fmin, fcount, err := f.FieldMin(filter, bitDepth) + fmin, fcount, err := f.Min(filter, bitDepth) if err != nil { return min, count, err } @@ -387,7 +387,7 @@ func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) { // max returns the max and count of a field. func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) { for _, f := range v.Fragments() { - fmax, fcount, err := f.FieldMax(filter, bitDepth) + fmax, fcount, err := f.Max(filter, bitDepth) if err != nil { return max, count, err } @@ -403,7 +403,7 @@ func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) { func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { r := NewRow() for _, frag := range v.Fragments() { - other, err := frag.FieldRange(op, bitDepth, predicate) + other, err := frag.RangeOp(op, bitDepth, predicate) if err != nil { return nil, err } @@ -417,7 +417,7 @@ func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, err func (v *View) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { r := NewRow() for _, frag := range v.Fragments() { - other, err := frag.FieldRangeBetween(bitDepth, predicateMin, predicateMax) + other, err := frag.RangeBetween(bitDepth, predicateMin, predicateMax) if err != nil { return nil, err } From 81a994987f12eb0ea5f13789babbb1ecf198c229 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 13:02:26 -0500 Subject: [PATCH 28/78] fixing some comments --- executor.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/executor.go b/executor.go index 98982606d..87e07c63e 100644 --- a/executor.go +++ b/executor.go @@ -790,12 +790,12 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, fieldName, cond = k, vv } - // EQ null (not implemented: flip frag.FieldNotNull with max ColumnID) - // NEQ null frag.FieldNotNull() - // BETWEEN a,b(in) BETWEEN/frag.FieldRangeBetween() - // BETWEEN a,b(out) BETWEEN/frag.FieldNotNull() - // EQ frag.FieldRange - // NEQ frag.FieldRange + // EQ null (not implemented: flip frag.NotNull with max ColumnID) + // NEQ null frag.NotNull() + // BETWEEN a,b(in) BETWEEN/frag.RangeBetween() + // BETWEEN a,b(out) BETWEEN/frag.NotNull() + // EQ frag.RangeOp + // NEQ frag.RangeOp // Handle `!= null`. if cond.Op == pql.NEQ && cond.Value == nil { @@ -826,7 +826,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // The reason we don't just call: - // return f.FieldRangeBetween(fieldName, predicates[0], predicates[1]) + // return f.RangeBetween(fieldName, predicates[0], predicates[1]) // here is because we need the call to be slice-specific. // Find field. From 50f8ea39213d1951cc7a798a1e0d6a3b4263e38f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 13:58:28 -0500 Subject: [PATCH 29/78] remove CreateField and DeleteField from API --- api.go | 65 ---- apimethod_string.go | 4 +- broadcast.go | 10 - internal/private.pb.go | 682 +++++++---------------------------------- internal/private.proto | 12 - server.go | 11 - 6 files changed, 113 insertions(+), 671 deletions(-) diff --git a/api.go b/api.go index 6f6123f9b..e528a4b21 100644 --- a/api.go +++ b/api.go @@ -488,67 +488,6 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.Holder.Schema() } -// CreateField creates a new BSI field in the given index and frame. -func (api *API) CreateField(ctx context.Context, indexName string, frameName string, bsig *bsiGroup) error { - if err := api.validate(apiCreateField); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Retrieve frame by name. - f := api.Holder.Frame(indexName, frameName) - if f == nil { - return ErrFrameNotFound - } - - // Create new bsiGroup. - if err := f.createBSIGroup(bsig); err != nil { - return errors.Wrap(err, "creating bsigroup") - } - - // Send the create bsigroup message to all nodes. - err := api.Broadcaster.SendSync( - &internal.CreateBSIGroupMessage{ - Index: indexName, - Frame: frameName, - BSIGroup: encodeBSIGroup(bsig), - }) - if err != nil { - api.Logger.Printf("problem sending CreateField message: %s", err) - } - return errors.Wrap(err, "sending CreateField message") -} - -// TODO: remove this from the API -// DeleteField deletes the given field. -func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { - if err := api.validate(apiDeleteField); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Retrieve frame by name. - f := api.Holder.Frame(indexName, frameName) - if f == nil { - return ErrFrameNotFound - } - - // Delete field. - if err := f.deleteBSIGroupAndView(fieldName); err != nil { - return errors.Wrap(err, "deleting field") - } - - // Send the delete field message to all nodes. - err := api.Broadcaster.SendSync( - &internal.DeleteBSIGroupMessage{ - Index: indexName, - Frame: frameName, - BSIGroup: fieldName, - }) - if err != nil { - api.Logger.Printf("problem sending DeleteField message: %s", err) - } - return errors.Wrap(err, "sending DeleteField message") -} - // Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { if err := api.validate(apiViews); err != nil { @@ -851,10 +790,8 @@ type apiMethod int // API validation constants. const ( apiClusterMessage apiMethod = iota - apiCreateField apiCreateFrame apiCreateIndex - apiDeleteField apiDeleteFrame apiDeleteIndex apiDeleteView @@ -896,10 +833,8 @@ var methodsResizing = map[apiMethod]struct{}{ } var methodsNormal = map[apiMethod]struct{}{ - apiCreateField: struct{}{}, apiCreateFrame: struct{}{}, apiCreateIndex: struct{}{}, - apiDeleteField: struct{}{}, apiDeleteFrame: struct{}{}, apiDeleteIndex: struct{}{}, apiDeleteView: struct{}{}, diff --git a/apimethod_string.go b/apimethod_string.go index 0fc3822d5..ce119196f 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -4,9 +4,9 @@ package pilosa import "fmt" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews" +const _apiMethod_name = "apiClusterMessageapiCreateFrameapiCreateIndexapiDeleteFrameapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 87, 101, 114, 126, 146, 163, 179, 188, 202, 210, 226, 244, 252, 272, 285, 299, 316, 329, 349, 357} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 151, 160, 174, 182, 198, 216, 224, 244, 257, 271, 288, 301, 321, 329} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/broadcast.go b/broadcast.go index 0840e8c92..933ca05a5 100644 --- a/broadcast.go +++ b/broadcast.go @@ -127,8 +127,6 @@ const ( MessageTypeDeleteFrame MessageTypeCreateView MessageTypeDeleteView - MessageTypeCreateBSIGroup - MessageTypeDeleteBSIGroup MessageTypeClusterStatus MessageTypeResizeInstruction MessageTypeResizeInstructionComplete @@ -157,10 +155,6 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeCreateView case *internal.DeleteViewMessage: typ = MessageTypeDeleteView - case *internal.CreateBSIGroupMessage: - typ = MessageTypeCreateBSIGroup - case *internal.DeleteBSIGroupMessage: - typ = MessageTypeDeleteBSIGroup case *internal.ClusterStatus: typ = MessageTypeClusterStatus case *internal.ResizeInstruction: @@ -207,10 +201,6 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.CreateViewMessage{} case MessageTypeDeleteView: m = &internal.DeleteViewMessage{} - case MessageTypeCreateBSIGroup: - m = &internal.CreateBSIGroupMessage{} - case MessageTypeDeleteBSIGroup: - m = &internal.DeleteBSIGroupMessage{} case MessageTypeClusterStatus: m = &internal.ClusterStatus{} case MessageTypeResizeInstruction: diff --git a/internal/private.pb.go b/internal/private.pb.go index 4e1aa8589..a4f9e0431 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -21,8 +21,6 @@ CreateIndexMessage CreateFrameMessage DeleteFrameMessage - CreateBSIGroupMessage - DeleteBSIGroupMessage Frame Schema Index @@ -366,70 +364,6 @@ func (m *DeleteFrameMessage) GetFrame() string { return "" } -type CreateBSIGroupMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - BSIGroup *BSIGroup `protobuf:"bytes,3,opt,name=BSIGroup" json:"BSIGroup,omitempty"` -} - -func (m *CreateBSIGroupMessage) Reset() { *m = CreateBSIGroupMessage{} } -func (m *CreateBSIGroupMessage) String() string { return proto.CompactTextString(m) } -func (*CreateBSIGroupMessage) ProtoMessage() {} -func (*CreateBSIGroupMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } - -func (m *CreateBSIGroupMessage) GetIndex() string { - if m != nil { - return m.Index - } - return "" -} - -func (m *CreateBSIGroupMessage) GetFrame() string { - if m != nil { - return m.Frame - } - return "" -} - -func (m *CreateBSIGroupMessage) GetBSIGroup() *BSIGroup { - if m != nil { - return m.BSIGroup - } - return nil -} - -type DeleteBSIGroupMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - BSIGroup string `protobuf:"bytes,3,opt,name=BSIGroup,proto3" json:"BSIGroup,omitempty"` -} - -func (m *DeleteBSIGroupMessage) Reset() { *m = DeleteBSIGroupMessage{} } -func (m *DeleteBSIGroupMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteBSIGroupMessage) ProtoMessage() {} -func (*DeleteBSIGroupMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } - -func (m *DeleteBSIGroupMessage) GetIndex() string { - if m != nil { - return m.Index - } - return "" -} - -func (m *DeleteBSIGroupMessage) GetFrame() string { - if m != nil { - return m.Frame - } - return "" -} - -func (m *DeleteBSIGroupMessage) GetBSIGroup() string { - if m != nil { - return m.BSIGroup - } - return "" -} - type Frame struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` @@ -439,7 +373,7 @@ type Frame struct { func (m *Frame) Reset() { *m = Frame{} } func (m *Frame) String() string { return proto.CompactTextString(m) } func (*Frame) ProtoMessage() {} -func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } +func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } func (m *Frame) GetName() string { if m != nil { @@ -469,7 +403,7 @@ type Schema struct { func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} -func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } +func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } func (m *Schema) GetIndexes() []*Index { if m != nil { @@ -486,7 +420,7 @@ type Index struct { func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} -func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } func (m *Index) GetName() string { if m != nil { @@ -511,7 +445,7 @@ type URI struct { func (m *URI) Reset() { *m = URI{} } func (m *URI) String() string { return proto.CompactTextString(m) } func (*URI) ProtoMessage() {} -func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } +func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } func (m *URI) GetScheme() string { if m != nil { @@ -543,7 +477,7 @@ type Node struct { func (m *Node) Reset() { *m = Node{} } func (m *Node) String() string { return proto.CompactTextString(m) } func (*Node) ProtoMessage() {} -func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } +func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } func (m *Node) GetID() string { if m != nil { @@ -574,7 +508,7 @@ type NodeStateMessage struct { func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } func (*NodeStateMessage) ProtoMessage() {} -func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } +func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } func (m *NodeStateMessage) GetNodeID() string { if m != nil { @@ -598,7 +532,7 @@ type NodeEventMessage struct { func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } func (*NodeEventMessage) ProtoMessage() {} -func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } +func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } func (m *NodeEventMessage) GetEvent() uint32 { if m != nil { @@ -623,7 +557,7 @@ type NodeStatus struct { func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } func (m *NodeStatus) GetNode() *Node { if m != nil { @@ -655,7 +589,7 @@ type ClusterStatus struct { func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } func (m *ClusterStatus) GetClusterID() string { if m != nil { @@ -688,7 +622,7 @@ type BSIGroup struct { func (m *BSIGroup) Reset() { *m = BSIGroup{} } func (m *BSIGroup) String() string { return proto.CompactTextString(m) } func (*BSIGroup) ProtoMessage() {} -func (*BSIGroup) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (*BSIGroup) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } func (m *BSIGroup) GetName() string { if m != nil { @@ -727,7 +661,7 @@ type CreateViewMessage struct { func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} -func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } +func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } func (m *CreateViewMessage) GetIndex() string { if m != nil { @@ -759,7 +693,7 @@ type DeleteViewMessage struct { func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -794,7 +728,7 @@ type ResizeInstruction struct { func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -849,7 +783,7 @@ type ResizeSource struct { func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } func (m *ResizeSource) GetNode() *Node { if m != nil { @@ -896,7 +830,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{28} + return fileDescriptorPrivate, []int{26} } func (m *ResizeInstructionComplete) GetJobID() int64 { @@ -927,7 +861,7 @@ type SetCoordinatorMessage struct { func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} -func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } +func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } func (m *SetCoordinatorMessage) GetNew() *Node { if m != nil { @@ -943,7 +877,7 @@ type UpdateCoordinatorMessage struct { func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessage{} } func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*UpdateCoordinatorMessage) ProtoMessage() {} -func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } +func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } func (m *UpdateCoordinatorMessage) GetNew() *Node { if m != nil { @@ -960,7 +894,7 @@ type Topology struct { func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } func (m *Topology) GetClusterID() string { if m != nil { @@ -982,7 +916,7 @@ type RecalculateCaches struct { func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} -func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{32} } +func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") @@ -997,8 +931,6 @@ func init() { proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") - proto.RegisterType((*CreateBSIGroupMessage)(nil), "internal.CreateBSIGroupMessage") - proto.RegisterType((*DeleteBSIGroupMessage)(nil), "internal.DeleteBSIGroupMessage") proto.RegisterType((*Frame)(nil), "internal.Frame") proto.RegisterType((*Schema)(nil), "internal.Schema") proto.RegisterType((*Index)(nil), "internal.Index") @@ -1436,82 +1368,6 @@ func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *CreateBSIGroupMessage) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *CreateBSIGroupMessage) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Frame) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) - } - if m.BSIGroup != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.BSIGroup.Size())) - n9, err := m.BSIGroup.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n9 - } - return i, nil -} - -func (m *DeleteBSIGroupMessage) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *DeleteBSIGroupMessage) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Frame) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) - } - if len(m.BSIGroup) > 0 { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.BSIGroup))) - i += copy(dAtA[i:], m.BSIGroup) - } - return i, nil -} - func (m *Frame) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1537,11 +1393,11 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n10, err := m.Meta.MarshalTo(dAtA[i:]) + n9, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n9 } if len(m.Views) > 0 { for _, s := range m.Views { @@ -1687,11 +1543,11 @@ func (m *Node) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n11, err := m.URI.MarshalTo(dAtA[i:]) + n10, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n11 + i += n10 } if m.IsCoordinator { dAtA[i] = 0x18 @@ -1760,11 +1616,11 @@ func (m *NodeEventMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n12, err := m.Node.MarshalTo(dAtA[i:]) + n11, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n12 + i += n11 } return i, nil } @@ -1788,31 +1644,31 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n13, err := m.Node.MarshalTo(dAtA[i:]) + n12, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n12 } if m.MaxSlices != nil { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size())) - n14, err := m.MaxSlices.MarshalTo(dAtA[i:]) + n13, err := m.MaxSlices.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n14 + i += n13 } if m.Schema != nil { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n15, err := m.Schema.MarshalTo(dAtA[i:]) + n14, err := m.Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n15 + i += n14 } return i, nil } @@ -1995,21 +1851,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n16, err := m.Node.MarshalTo(dAtA[i:]) + n15, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n16 + i += n15 } if m.Coordinator != nil { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Coordinator.Size())) - n17, err := m.Coordinator.MarshalTo(dAtA[i:]) + n16, err := m.Coordinator.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n17 + i += n16 } if len(m.Sources) > 0 { for _, msg := range m.Sources { @@ -2027,21 +1883,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n18, err := m.Schema.MarshalTo(dAtA[i:]) + n17, err := m.Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n18 + i += n17 } if m.ClusterStatus != nil { dAtA[i] = 0x32 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) - n19, err := m.ClusterStatus.MarshalTo(dAtA[i:]) + n18, err := m.ClusterStatus.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n19 + i += n18 } return i, nil } @@ -2065,11 +1921,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n20, err := m.Node.MarshalTo(dAtA[i:]) + n19, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n20 + i += n19 } if len(m.Index) > 0 { dAtA[i] = 0x12 @@ -2121,11 +1977,11 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n21, err := m.Node.MarshalTo(dAtA[i:]) + n20, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n21 + i += n20 } if len(m.Error) > 0 { dAtA[i] = 0x1a @@ -2155,11 +2011,11 @@ func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n22, err := m.New.MarshalTo(dAtA[i:]) + n21, err := m.New.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n22 + i += n21 } return i, nil } @@ -2183,11 +2039,11 @@ func (m *UpdateCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n23, err := m.New.MarshalTo(dAtA[i:]) + n22, err := m.New.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n23 + i += n22 } return i, nil } @@ -2459,42 +2315,6 @@ func (m *DeleteFrameMessage) Size() (n int) { return n } -func (m *CreateBSIGroupMessage) Size() (n int) { - var l int - _ = l - l = len(m.Index) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - l = len(m.Frame) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - if m.BSIGroup != nil { - l = m.BSIGroup.Size() - n += 1 + l + sovPrivate(uint64(l)) - } - return n -} - -func (m *DeleteBSIGroupMessage) Size() (n int) { - var l int - _ = l - l = len(m.Index) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - l = len(m.Frame) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - l = len(m.BSIGroup) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - return n -} - func (m *Frame) Size() (n int) { var l int _ = l @@ -4304,284 +4124,6 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *CreateBSIGroupMessage) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: CreateBSIGroupMessage: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: CreateBSIGroupMessage: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Index = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Frame = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field BSIGroup", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.BSIGroup == nil { - m.BSIGroup = &BSIGroup{} - } - if err := m.BSIGroup.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *DeleteBSIGroupMessage) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: DeleteBSIGroupMessage: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: DeleteBSIGroupMessage: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Index = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Frame = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field BSIGroup", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.BSIGroup = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *Frame) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -7075,70 +6617,68 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1038 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0x1b, 0xc5, - 0x1b, 0xff, 0xaf, 0x77, 0xed, 0xd8, 0x4f, 0xfe, 0x0e, 0xc9, 0x94, 0x84, 0x6d, 0x85, 0x82, 0x19, - 0x55, 0x22, 0x70, 0xb0, 0x4a, 0x7b, 0xe1, 0xad, 0x52, 0x14, 0x3b, 0xc0, 0x22, 0x12, 0xc1, 0x6c, - 0xd2, 0x03, 0x12, 0x42, 0x53, 0x7b, 0xd4, 0xae, 0xb2, 0xde, 0x31, 0xbb, 0xb3, 0x49, 0xdc, 0x03, - 0x57, 0xb8, 0x70, 0x47, 0xdc, 0xf8, 0x36, 0x1c, 0xf9, 0x08, 0x28, 0x7c, 0x11, 0x34, 0xcf, 0xcc, - 0xbe, 0xc4, 0x2f, 0x4d, 0x95, 0x72, 0x9b, 0xe7, 0xfd, 0xed, 0xf7, 0xcc, 0x0c, 0x74, 0xa7, 0x69, - 0x74, 0xce, 0x95, 0xe8, 0x4f, 0x53, 0xa9, 0x24, 0x69, 0x47, 0x89, 0x12, 0x69, 0xc2, 0x63, 0xba, - 0x0e, 0x9d, 0x20, 0x19, 0x8b, 0xcb, 0x23, 0xa1, 0x38, 0xfd, 0xc3, 0x81, 0xce, 0xe7, 0x29, 0x9f, - 0x08, 0x4d, 0x91, 0xb7, 0xa1, 0x33, 0xe0, 0xa3, 0xe7, 0xe2, 0x64, 0x36, 0x15, 0xbe, 0xdb, 0x73, - 0xf6, 0x3a, 0xac, 0x62, 0x94, 0xd2, 0x30, 0x7a, 0x21, 0x7c, 0xaf, 0xe7, 0xec, 0x75, 0x59, 0xc5, - 0x20, 0x3d, 0x58, 0x3f, 0x89, 0x26, 0xe2, 0xdb, 0x9c, 0x27, 0x2a, 0x9f, 0xf8, 0x4d, 0xb4, 0xae, - 0xb3, 0x08, 0x01, 0x0f, 0x1d, 0xb7, 0x51, 0x84, 0x67, 0xb2, 0x09, 0xee, 0x51, 0x94, 0xf8, 0x9d, - 0x9e, 0xb3, 0xe7, 0x32, 0x7d, 0x44, 0x0e, 0xbf, 0xf4, 0xc1, 0x72, 0xf8, 0x25, 0xa5, 0xb0, 0x11, - 0x4c, 0xa6, 0x32, 0x55, 0x4c, 0x64, 0x53, 0x99, 0x64, 0x68, 0x75, 0x98, 0xa6, 0xbe, 0x83, 0x8e, - 0xf4, 0x91, 0xfe, 0x04, 0x9b, 0x07, 0xb1, 0x1c, 0x9d, 0x0d, 0xb9, 0xe2, 0x4c, 0xfc, 0x98, 0x8b, - 0x4c, 0x91, 0x37, 0xa1, 0x89, 0x85, 0x5a, 0x3d, 0x43, 0x68, 0x2e, 0x16, 0xec, 0x37, 0x0c, 0x17, - 0x09, 0xcd, 0x45, 0x7b, 0xac, 0xda, 0x63, 0x86, 0xd0, 0xdc, 0x30, 0x8e, 0x46, 0xa6, 0x5a, 0x8f, - 0x19, 0x42, 0xd7, 0xf1, 0x24, 0x12, 0x17, 0xb6, 0x44, 0x3c, 0xd3, 0x00, 0xb6, 0x6a, 0xf1, 0x6d, - 0x9a, 0x3b, 0xd0, 0x62, 0xf2, 0x22, 0x18, 0x66, 0xbe, 0xd3, 0x73, 0xf7, 0x3c, 0x66, 0x29, 0x6c, - 0xa4, 0x8c, 0xf3, 0x49, 0xa2, 0x45, 0x0d, 0x14, 0x55, 0x0c, 0x7a, 0x17, 0x9a, 0xd8, 0x55, 0x5d, - 0x65, 0x65, 0xab, 0x8f, 0xf4, 0x67, 0x07, 0x3a, 0x47, 0xfc, 0x12, 0xd3, 0xc8, 0xc8, 0x63, 0x68, - 0x87, 0x8a, 0x27, 0x63, 0x9e, 0x8e, 0x51, 0x69, 0xfd, 0xe1, 0xbb, 0xfd, 0x62, 0xca, 0xfd, 0x52, - 0xad, 0x5f, 0xe8, 0x1c, 0x26, 0x2a, 0x9d, 0xb1, 0xd2, 0xe4, 0xde, 0xa7, 0xd0, 0xbd, 0x26, 0xd2, - 0xf1, 0xce, 0xc4, 0xac, 0xe8, 0xea, 0x99, 0x98, 0xe9, 0xfa, 0xcf, 0x79, 0x9c, 0x9b, 0x5e, 0x79, - 0xcc, 0x10, 0x9f, 0x34, 0x3e, 0x72, 0xe8, 0x3e, 0x90, 0x41, 0x2a, 0xb8, 0x12, 0x18, 0xe4, 0x48, - 0x64, 0x19, 0x7f, 0x26, 0x56, 0x77, 0xdc, 0x74, 0xb1, 0x51, 0xeb, 0x22, 0xfd, 0x00, 0xc8, 0x50, - 0xc4, 0x42, 0x09, 0x0b, 0xc6, 0x97, 0x78, 0xa0, 0x61, 0x11, 0xed, 0x66, 0x5d, 0xf2, 0x1e, 0x78, - 0x1a, 0xcb, 0x18, 0x6c, 0xfd, 0xe1, 0x9d, 0xaa, 0x23, 0x25, 0xe8, 0x19, 0x2a, 0xd0, 0xa8, 0x70, - 0x6a, 0xf1, 0x7f, 0x43, 0x09, 0x4b, 0x40, 0x53, 0x84, 0x72, 0xe7, 0x43, 0x95, 0x1b, 0x65, 0x43, - 0xed, 0x17, 0xb5, 0xde, 0x36, 0x14, 0xcd, 0x60, 0xdb, 0x24, 0x7b, 0x10, 0x06, 0x5f, 0xa4, 0x32, - 0x9f, 0xde, 0x26, 0xdf, 0x3e, 0xb4, 0x0b, 0x73, 0x9b, 0x33, 0xa9, 0x72, 0x2e, 0x24, 0xac, 0xd4, - 0xa1, 0x3f, 0xc0, 0xb6, 0x49, 0xfb, 0x75, 0x82, 0xde, 0x9b, 0x0b, 0xda, 0xa9, 0x05, 0xf8, 0xce, - 0x5a, 0xe8, 0x95, 0x3a, 0xd6, 0x96, 0xc6, 0x1f, 0x9e, 0x57, 0x0f, 0x72, 0xae, 0xbb, 0x3a, 0xae, - 0xde, 0xc1, 0xcc, 0x77, 0x7b, 0xae, 0x8e, 0x8b, 0x04, 0x7d, 0x04, 0xad, 0x70, 0xf4, 0x5c, 0x4c, - 0x38, 0x79, 0x1f, 0xd6, 0x30, 0x41, 0x91, 0xd9, 0x35, 0x79, 0x63, 0x0e, 0x14, 0xac, 0x90, 0xd3, - 0xa1, 0x2d, 0x6c, 0x45, 0x42, 0x2d, 0x0c, 0x9d, 0xf9, 0xde, 0xbc, 0x1b, 0xe4, 0x33, 0x2b, 0xa6, - 0x87, 0xe0, 0x9e, 0xb2, 0x40, 0xaf, 0x3f, 0x66, 0x50, 0x78, 0xb1, 0x94, 0xf6, 0xfd, 0xa5, 0xcc, - 0x94, 0x6d, 0x13, 0x9e, 0x35, 0xef, 0x1b, 0x99, 0x2a, 0xec, 0x50, 0x97, 0xe1, 0x99, 0x7e, 0x0f, - 0xde, 0xb1, 0x1c, 0x0b, 0xb2, 0x01, 0x8d, 0x60, 0x68, 0x7d, 0x34, 0x82, 0x21, 0x79, 0x07, 0xdd, - 0xdb, 0xbe, 0x74, 0xab, 0x24, 0x4e, 0x59, 0xc0, 0x30, 0xf0, 0x7d, 0xe8, 0x06, 0xd9, 0x40, 0xca, - 0x74, 0x1c, 0x25, 0x5c, 0xc9, 0x14, 0xbd, 0xb6, 0xd9, 0x75, 0x26, 0xdd, 0x87, 0x4d, 0xed, 0x3e, - 0x54, 0x5c, 0x95, 0x90, 0xdc, 0x81, 0x96, 0xe6, 0x95, 0xe1, 0x2c, 0x85, 0x2b, 0xac, 0xf5, 0x8a, - 0xd1, 0x22, 0x41, 0xbf, 0x36, 0x1e, 0x0e, 0xcf, 0x45, 0xa2, 0x6a, 0xd0, 0x40, 0x1a, 0x1d, 0x74, - 0x99, 0x21, 0x08, 0x35, 0xa5, 0xd8, 0x9c, 0x37, 0xaa, 0x9c, 0x35, 0x97, 0xa1, 0x8c, 0xfe, 0xea, - 0x00, 0x14, 0x09, 0xe5, 0x59, 0x69, 0xe2, 0xac, 0x36, 0x21, 0x1f, 0xd6, 0xae, 0xc3, 0x45, 0x9c, - 0x94, 0x22, 0x56, 0xbb, 0x34, 0xf7, 0x0a, 0x58, 0xd8, 0x0d, 0xd8, 0xac, 0xf4, 0x0d, 0xdf, 0x8e, - 0x49, 0xdf, 0x0f, 0xdd, 0x41, 0x9c, 0x67, 0x4a, 0xa4, 0x36, 0x23, 0x7d, 0x6d, 0x1b, 0x46, 0xd9, - 0x9f, 0x8a, 0xb1, 0xbc, 0x45, 0xe4, 0x3e, 0x34, 0x75, 0xa6, 0x06, 0x9b, 0x8b, 0x65, 0x18, 0x21, - 0x7d, 0x52, 0xed, 0xc8, 0x52, 0xe4, 0x15, 0x2f, 0x67, 0x63, 0xf1, 0xe5, 0x74, 0x17, 0x5e, 0x4e, - 0xaf, 0x7a, 0x39, 0x43, 0xd8, 0x32, 0xb7, 0x86, 0x5e, 0x89, 0xdb, 0x2c, 0x6f, 0xf1, 0xd4, 0xb9, - 0xb5, 0xa7, 0x2e, 0x84, 0x2d, 0x73, 0x2b, 0xfc, 0x97, 0x4e, 0x7f, 0x6f, 0xc0, 0x16, 0x13, 0x59, - 0xf4, 0x42, 0x04, 0x49, 0xa6, 0xd2, 0x7c, 0xa4, 0x22, 0x99, 0x68, 0xfb, 0xaf, 0xe4, 0x53, 0xdb, - 0x6d, 0x97, 0x19, 0xe2, 0x55, 0xc0, 0x44, 0x1e, 0xc0, 0xfa, 0xfc, 0x02, 0x2c, 0xaa, 0xd6, 0x55, - 0xc8, 0x03, 0x58, 0x0b, 0x65, 0x9e, 0x8e, 0xca, 0xf5, 0xde, 0xa9, 0xb4, 0x4d, 0x66, 0x46, 0xcc, - 0x0a, 0xb5, 0x1a, 0x94, 0x9a, 0x2f, 0x87, 0x12, 0x79, 0x3c, 0x07, 0x25, 0xbf, 0x85, 0x06, 0x6f, - 0x55, 0x06, 0xd7, 0xc4, 0xec, 0xba, 0x36, 0xfd, 0xc5, 0x81, 0xff, 0xd7, 0x53, 0x78, 0xa5, 0xdd, - 0x28, 0x27, 0xd2, 0x58, 0x3a, 0x11, 0x77, 0xd9, 0x44, 0xbc, 0x6a, 0x22, 0xd5, 0xab, 0xdd, 0xac, - 0xbf, 0xda, 0x67, 0x70, 0x77, 0x61, 0x4c, 0x03, 0x39, 0x99, 0x6a, 0x3c, 0xbc, 0xc6, 0xb8, 0xf4, - 0xad, 0x91, 0xa6, 0x76, 0x50, 0x1d, 0x66, 0x08, 0xfa, 0x31, 0x6c, 0x87, 0x42, 0xd5, 0x86, 0x54, - 0xa0, 0xad, 0x07, 0xee, 0xb1, 0xb8, 0x58, 0x51, 0xbe, 0x16, 0xd1, 0xcf, 0xc0, 0x3f, 0x9d, 0x8e, - 0xb9, 0x12, 0xb7, 0xb2, 0x3e, 0x80, 0xf6, 0x89, 0x9c, 0xca, 0x58, 0x3e, 0x9b, 0xdd, 0xb0, 0xf5, - 0x3e, 0xac, 0x99, 0x2b, 0xd2, 0x7c, 0xe4, 0x3a, 0xac, 0x20, 0xe9, 0x1d, 0x0d, 0xe8, 0x11, 0x8f, - 0x47, 0x79, 0xac, 0xd3, 0xd0, 0x3f, 0xba, 0xec, 0x60, 0xf3, 0xcf, 0xab, 0x5d, 0xe7, 0xaf, 0xab, - 0x5d, 0xe7, 0xef, 0xab, 0x5d, 0xe7, 0xb7, 0x7f, 0x76, 0xff, 0xf7, 0xb4, 0x85, 0xdf, 0xf3, 0x47, - 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xa3, 0xc6, 0x83, 0xfe, 0xaf, 0x0b, 0x00, 0x00, + // 1002 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcd, 0x6f, 0x1b, 0x45, + 0x14, 0x67, 0xbd, 0x6b, 0xc7, 0x7e, 0xc1, 0xc1, 0x99, 0x42, 0xd8, 0x22, 0x14, 0xcc, 0xa8, 0x12, + 0x86, 0x43, 0x54, 0xda, 0x0b, 0x5f, 0x95, 0xa2, 0xd8, 0x01, 0x16, 0x91, 0x08, 0x66, 0x93, 0x1e, + 0x90, 0x38, 0x4c, 0xed, 0x51, 0xbb, 0xca, 0x7a, 0xc7, 0xec, 0xce, 0x26, 0x71, 0x0f, 0x5c, 0xe1, + 0xc2, 0x1d, 0x71, 0xe3, 0xbf, 0xe1, 0xc8, 0x9f, 0x80, 0xc2, 0x3f, 0x82, 0xe6, 0xcd, 0xec, 0x47, + 0xfc, 0xd1, 0x54, 0xa1, 0xb7, 0x79, 0xbf, 0xf7, 0xfd, 0xde, 0x6f, 0x76, 0x16, 0xba, 0xb3, 0x34, + 0x3a, 0xe7, 0x4a, 0xec, 0xcd, 0x52, 0xa9, 0x24, 0x69, 0x47, 0x89, 0x12, 0x69, 0xc2, 0x63, 0xba, + 0x09, 0x9d, 0x20, 0x99, 0x88, 0xcb, 0x23, 0xa1, 0x38, 0xfd, 0xd3, 0x81, 0xce, 0x97, 0x29, 0x9f, + 0x0a, 0x2d, 0x91, 0x77, 0xa1, 0x33, 0xe4, 0xe3, 0x67, 0xe2, 0x64, 0x3e, 0x13, 0xbe, 0xdb, 0x77, + 0x06, 0x1d, 0x56, 0x01, 0xa5, 0x36, 0x8c, 0x9e, 0x0b, 0xdf, 0xeb, 0x3b, 0x83, 0x2e, 0xab, 0x00, + 0xd2, 0x87, 0xcd, 0x93, 0x68, 0x2a, 0xbe, 0xcf, 0x79, 0xa2, 0xf2, 0xa9, 0xdf, 0x44, 0xef, 0x3a, + 0x44, 0x08, 0x78, 0x18, 0xb8, 0x8d, 0x2a, 0x3c, 0x93, 0x1e, 0xb8, 0x47, 0x51, 0xe2, 0x77, 0xfa, + 0xce, 0xc0, 0x65, 0xfa, 0x88, 0x08, 0xbf, 0xf4, 0xc1, 0x22, 0xfc, 0x92, 0x52, 0xd8, 0x0a, 0xa6, + 0x33, 0x99, 0x2a, 0x26, 0xb2, 0x99, 0x4c, 0x32, 0xf4, 0x3a, 0x4c, 0x53, 0xdf, 0xc1, 0x40, 0xfa, + 0x48, 0x7f, 0x86, 0xde, 0x41, 0x2c, 0xc7, 0x67, 0x23, 0xae, 0x38, 0x13, 0x3f, 0xe5, 0x22, 0x53, + 0xe4, 0x4d, 0x68, 0x62, 0xa3, 0xd6, 0xce, 0x08, 0x1a, 0xc5, 0x86, 0xfd, 0x86, 0x41, 0x51, 0xd0, + 0x28, 0xfa, 0x63, 0xd7, 0x1e, 0x33, 0x82, 0x46, 0xc3, 0x38, 0x1a, 0x9b, 0x6e, 0x3d, 0x66, 0x04, + 0xdd, 0xc7, 0xe3, 0x48, 0x5c, 0xd8, 0x16, 0xf1, 0x4c, 0x03, 0xd8, 0xae, 0xe5, 0xb7, 0x65, 0xee, + 0x40, 0x8b, 0xc9, 0x8b, 0x60, 0x94, 0xf9, 0x4e, 0xdf, 0x1d, 0x78, 0xcc, 0x4a, 0x38, 0x48, 0x19, + 0xe7, 0xd3, 0x44, 0xab, 0x1a, 0xa8, 0xaa, 0x00, 0x7a, 0x17, 0x9a, 0x38, 0x55, 0xdd, 0x65, 0xe5, + 0xab, 0x8f, 0xf4, 0x17, 0x07, 0x3a, 0x47, 0xfc, 0x12, 0xcb, 0xc8, 0xc8, 0x23, 0x68, 0x87, 0x8a, + 0x27, 0x13, 0x9e, 0x4e, 0xd0, 0x68, 0xf3, 0xc1, 0xfb, 0x7b, 0xc5, 0x96, 0xf7, 0x4a, 0xb3, 0xbd, + 0xc2, 0xe6, 0x30, 0x51, 0xe9, 0x9c, 0x95, 0x2e, 0xef, 0x7c, 0x0e, 0xdd, 0x6b, 0x2a, 0x9d, 0xef, + 0x4c, 0xcc, 0x8b, 0xa9, 0x9e, 0x89, 0xb9, 0xee, 0xff, 0x9c, 0xc7, 0xb9, 0x99, 0x95, 0xc7, 0x8c, + 0xf0, 0x59, 0xe3, 0x13, 0x87, 0xee, 0x03, 0x19, 0xa6, 0x82, 0x2b, 0x81, 0x49, 0x8e, 0x44, 0x96, + 0xf1, 0xa7, 0x62, 0xfd, 0xc4, 0xcd, 0x14, 0x1b, 0xb5, 0x29, 0xd2, 0x8f, 0x80, 0x8c, 0x44, 0x2c, + 0x94, 0xb0, 0x64, 0x7c, 0x41, 0x04, 0x1a, 0x16, 0xd9, 0x6e, 0xb6, 0x25, 0x1f, 0x80, 0xa7, 0xb9, + 0x8c, 0xc9, 0x36, 0x1f, 0xdc, 0xa9, 0x26, 0x52, 0x92, 0x9e, 0xa1, 0x01, 0x8d, 0x8a, 0xa0, 0x96, + 0xff, 0x37, 0xb4, 0xb0, 0x82, 0x34, 0x45, 0x2a, 0x77, 0x31, 0x55, 0x79, 0xa3, 0x6c, 0xaa, 0xfd, + 0xa2, 0xd7, 0xdb, 0xa6, 0xa2, 0x3f, 0x58, 0x54, 0x93, 0xef, 0x58, 0x6b, 0x8d, 0x0f, 0x9e, 0xd7, + 0xb7, 0xbc, 0x50, 0x87, 0x8e, 0xad, 0xd9, 0x9a, 0xf9, 0x6e, 0xdf, 0xd5, 0xb1, 0x51, 0xa0, 0x0f, + 0xa1, 0x15, 0x8e, 0x9f, 0x89, 0x29, 0x27, 0x1f, 0xc2, 0x06, 0x16, 0x21, 0x32, 0x4b, 0xa8, 0x37, + 0x16, 0xc6, 0xc7, 0x0a, 0x3d, 0x1d, 0xd9, 0xe2, 0xd7, 0x14, 0xd4, 0xc2, 0xd4, 0x99, 0xef, 0x2d, + 0x86, 0x41, 0x9c, 0x59, 0x35, 0x3d, 0x04, 0xf7, 0x94, 0x05, 0xfa, 0xa2, 0x60, 0x05, 0x45, 0x14, + 0x2b, 0xe9, 0xd8, 0x5f, 0xcb, 0x4c, 0xd9, 0x51, 0xe0, 0x59, 0x63, 0xdf, 0xc9, 0x54, 0xe1, 0xd0, + 0xbb, 0x0c, 0xcf, 0xf4, 0x47, 0xf0, 0x8e, 0xe5, 0x44, 0x90, 0x2d, 0x68, 0x04, 0x23, 0x1b, 0xa3, + 0x11, 0x8c, 0xc8, 0x7b, 0x18, 0xde, 0xce, 0xa5, 0x5b, 0x15, 0x71, 0xca, 0x02, 0x86, 0x89, 0xef, + 0x41, 0x37, 0xc8, 0x86, 0x52, 0xa6, 0x93, 0x28, 0xe1, 0x4a, 0xa6, 0x18, 0xb5, 0xcd, 0xae, 0x83, + 0x74, 0x1f, 0x7a, 0x3a, 0x7c, 0xa8, 0xb8, 0x2a, 0x97, 0xb7, 0x03, 0x2d, 0x8d, 0x95, 0xe9, 0xac, + 0x84, 0x64, 0xd7, 0x76, 0xc5, 0xfa, 0x50, 0xa0, 0xdf, 0x9a, 0x08, 0x87, 0xe7, 0x22, 0x51, 0xb5, + 0xf5, 0xa3, 0x8c, 0x01, 0xba, 0xcc, 0x08, 0x84, 0x9a, 0x56, 0x6c, 0xcd, 0x5b, 0x55, 0xcd, 0x1a, + 0x65, 0xa8, 0xa3, 0xbf, 0x39, 0x00, 0x45, 0x41, 0x79, 0x56, 0xba, 0x38, 0xeb, 0x5d, 0xc8, 0xc7, + 0xb5, 0x0f, 0xc7, 0x32, 0x4f, 0x4a, 0x15, 0xab, 0x7d, 0x5e, 0x06, 0x05, 0x2d, 0x2c, 0xbf, 0x7b, + 0x95, 0xbd, 0xc1, 0xed, 0x9a, 0xf4, 0x4d, 0xea, 0x0e, 0xe3, 0x3c, 0x53, 0x22, 0xb5, 0x15, 0xe9, + 0x0f, 0x9c, 0x01, 0xca, 0xf9, 0x54, 0xc0, 0xea, 0x11, 0x91, 0x7b, 0xd0, 0xd4, 0x95, 0x1a, 0x6e, + 0x2e, 0xb7, 0x61, 0x94, 0xf4, 0x31, 0xb4, 0x0f, 0xc2, 0xe0, 0xab, 0x54, 0xe6, 0xb3, 0x95, 0xcc, + 0x2b, 0xde, 0x98, 0xc6, 0xf2, 0x1b, 0xe3, 0x2e, 0xbd, 0x31, 0x5e, 0xf5, 0xc6, 0x84, 0xb0, 0x6d, + 0x3e, 0x06, 0xfa, 0x4a, 0xdc, 0xe6, 0x5b, 0x50, 0x3c, 0x0a, 0x6e, 0xed, 0x51, 0x08, 0x61, 0xdb, + 0x5c, 0xfb, 0x57, 0x19, 0xf4, 0x8f, 0x06, 0x6c, 0x33, 0x91, 0x45, 0xcf, 0x45, 0x90, 0x64, 0x2a, + 0xcd, 0xc7, 0x2a, 0x92, 0x89, 0xf6, 0xff, 0x46, 0x3e, 0xb1, 0xd3, 0x76, 0x99, 0x11, 0x5e, 0x86, + 0x4c, 0xe4, 0x3e, 0x6c, 0x2e, 0x5e, 0x80, 0x65, 0xd3, 0xba, 0x09, 0xb9, 0x0f, 0x1b, 0xa1, 0xcc, + 0xd3, 0x71, 0x79, 0xbd, 0x77, 0x2a, 0x6b, 0x53, 0x99, 0x51, 0xb3, 0xc2, 0xac, 0x46, 0xa5, 0xe6, + 0x8b, 0xa9, 0x44, 0x1e, 0x2d, 0x50, 0xc9, 0x6f, 0xa1, 0xc3, 0xdb, 0x95, 0xc3, 0x35, 0x35, 0xbb, + 0x6e, 0x4d, 0x7f, 0x75, 0xe0, 0xf5, 0x7a, 0x09, 0x2f, 0x75, 0x37, 0xca, 0x8d, 0x34, 0x56, 0x6e, + 0xc4, 0x5d, 0xb5, 0x11, 0xaf, 0xda, 0x48, 0xf5, 0xbe, 0x35, 0xeb, 0xef, 0xdb, 0x19, 0xdc, 0x5d, + 0x5a, 0xd3, 0x50, 0x4e, 0x67, 0x9a, 0x0f, 0xff, 0x63, 0x5d, 0xfa, 0xab, 0x91, 0xa6, 0x76, 0x51, + 0x1d, 0x66, 0x04, 0xfa, 0x29, 0xbc, 0x15, 0x0a, 0x55, 0x5b, 0x52, 0xc1, 0xb6, 0x3e, 0xb8, 0xc7, + 0xe2, 0x62, 0x4d, 0xfb, 0x5a, 0x45, 0xbf, 0x00, 0xff, 0x74, 0x36, 0xe1, 0x4a, 0xdc, 0xca, 0xfb, + 0x00, 0xda, 0x27, 0x72, 0x26, 0x63, 0xf9, 0x74, 0x7e, 0xc3, 0xad, 0xf7, 0x61, 0xc3, 0x7c, 0x22, + 0xcd, 0x2f, 0x4f, 0x87, 0x15, 0x22, 0xbd, 0xa3, 0x09, 0x3d, 0xe6, 0xf1, 0x38, 0x8f, 0x75, 0x19, + 0xfa, 0xdf, 0x27, 0x3b, 0xe8, 0xfd, 0x75, 0xb5, 0xeb, 0xfc, 0x7d, 0xb5, 0xeb, 0xfc, 0x73, 0xb5, + 0xeb, 0xfc, 0xfe, 0xef, 0xee, 0x6b, 0x4f, 0x5a, 0xf8, 0x23, 0xfb, 0xf0, 0xbf, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x93, 0x15, 0x15, 0x14, 0xd9, 0x0a, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index df004456f..d150b7802 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -64,18 +64,6 @@ message DeleteFrameMessage { string Frame = 2; } -message CreateBSIGroupMessage { - string Index = 1; - string Frame = 2; - BSIGroup BSIGroup = 3; -} - -message DeleteBSIGroupMessage { - string Index = 1; - string Frame = 2; - string BSIGroup = 3; -} - message Frame { string Name = 1; FrameMeta Meta = 2; diff --git a/server.go b/server.go index 1e4f5a421..02e56e9b0 100644 --- a/server.go +++ b/server.go @@ -470,17 +470,6 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err := idx.DeleteFrame(obj.Frame); err != nil { return err } - case *internal.CreateBSIGroupMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) - field := decodeBSIGroup(obj.BSIGroup) - if err := f.createBSIGroup(field); err != nil { - return err - } - case *internal.DeleteBSIGroupMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) - if err := f.deleteBSIGroupAndView(obj.BSIGroup); err != nil { - return err - } case *internal.CreateViewMessage: f := s.Holder.Frame(obj.Index, obj.Frame) if f == nil { From 386545c67cb57aacdc4160aa9e0f543b4642cc50 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 14:33:26 -0500 Subject: [PATCH 30/78] rename TopN field/filters to attrName/attrValues --- executor.go | 8 ++++---- executor_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/executor.go b/executor.go index 87e07c63e..413be36b4 100644 --- a/executor.go +++ b/executor.go @@ -552,7 +552,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) } - field, _ := c.Args["field"].(string) // TODO: rename this to something other than field + attrName, _ := c.Args["attrName"].(string) rowIDs, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) @@ -561,7 +561,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) } - filters, _ := c.Args["filters"].([]interface{}) + attrValues, _ := c.Args["attrValues"].([]interface{}) tanimotoThreshold, _, err := c.UintArg("tanimotoThreshold") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) @@ -600,8 +600,8 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca N: int(n), Src: src, RowIDs: rowIDs, - FilterName: field, - FilterValues: filters, + FilterName: attrName, + FilterValues: attrValues, MinThreshold: minThreshold, TanimotoThreshold: tanimotoThreshold, }) diff --git a/executor_test.go b/executor_test.go index 606646a92..d06a5027c 100644 --- a/executor_test.go +++ b/executor_test.go @@ -529,7 +529,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -552,7 +552,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,frame=f),frame="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, From 2802f6f469e52e6fcaf558d23a2275ededba7867 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 15:09:41 -0500 Subject: [PATCH 31/78] remove frame argument from Range() queries --- executor.go | 136 ++++++++++++++++++++--------------------------- executor_test.go | 36 ++++++------- 2 files changed, 74 insertions(+), 98 deletions(-) diff --git a/executor.go b/executor.go index 413be36b4..eee4fa0ee 100644 --- a/executor.go +++ b/executor.go @@ -176,8 +176,6 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame := c.Args["frame"]; frame == "" { return ValCount{}, errors.New("Sum(): frame required") - } else if field := c.Args["field"]; field == "" { - return ValCount{}, errors.New("Sum(): field required") } if len(c.Children) > 1 { @@ -211,8 +209,6 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame := c.Args["frame"]; frame == "" { return ValCount{}, errors.New("Min(): frame required") - } else if field := c.Args["field"]; field == "" { - return ValCount{}, errors.New("Min(): field required") } if len(c.Children) > 1 { @@ -246,8 +242,6 @@ func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, sl func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame := c.Args["frame"]; frame == "" { return ValCount{}, errors.New("Max(): frame required") - } else if field := c.Args["field"]; field == "" { - return ValCount{}, errors.New("Max(): field required") } if len(c.Children) > 1 { @@ -362,7 +356,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * } } -// executeSumCountSlice calculates the sum and count for fields on a slice. +// executeSumCountSlice calculates the sum and count for bsiGroups on a slice. func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { @@ -374,34 +368,33 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq } frameName, _ := c.Args["frame"].(string) - fieldName, _ := c.Args["field"].(string) frame := e.Holder.Frame(index, frameName) if frame == nil { return ValCount{}, nil } - field := frame.bsiGroup(fieldName) - if field == nil { + bsig := frame.bsiGroup(frameName) + if bsig == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) if fragment == nil { return ValCount{}, nil } - vsum, vcount, err := fragment.Sum(filter, field.BitDepth()) + vsum, vcount, err := fragment.Sum(filter, bsig.BitDepth()) if err != nil { return ValCount{}, errors.Wrap(err, "computing sum") } return ValCount{ - Val: int64(vsum) + (int64(vcount) * field.Min), + Val: int64(vsum) + (int64(vcount) * bsig.Min), Count: int64(vcount), }, nil } -// executeMinSlice calculates the min for fields on a slice. +// executeMinSlice calculates the min for bsiGroups on a slice. func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { @@ -413,34 +406,33 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal } frameName, _ := c.Args["frame"].(string) - fieldName, _ := c.Args["field"].(string) frame := e.Holder.Frame(index, frameName) if frame == nil { return ValCount{}, nil } - field := frame.bsiGroup(fieldName) - if field == nil { + bsig := frame.bsiGroup(frameName) + if bsig == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) if fragment == nil { return ValCount{}, nil } - fmin, fcount, err := fragment.Min(filter, field.BitDepth()) + fmin, fcount, err := fragment.Min(filter, bsig.BitDepth()) if err != nil { return ValCount{}, err } return ValCount{ - Val: int64(fmin) + field.Min, + Val: int64(fmin) + bsig.Min, Count: int64(fcount), }, nil } -// executeMaxSlice calculates the max for fields on a slice. +// executeMaxSlice calculates the max for bsiGroups on a slice. func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { @@ -452,29 +444,28 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal } frameName, _ := c.Args["frame"].(string) - fieldName, _ := c.Args["field"].(string) frame := e.Holder.Frame(index, frameName) if frame == nil { return ValCount{}, nil } - field := frame.bsiGroup(fieldName) - if field == nil { + bsig := frame.bsiGroup(frameName) + if bsig == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) if fragment == nil { return ValCount{}, nil } - fmax, fcount, err := fragment.Max(filter, field.BitDepth()) + fmax, fcount, err := fragment.Max(filter, bsig.BitDepth()) if err != nil { return ValCount{}, err } return ValCount{ - Val: int64(fmax) + field.Min, + Val: int64(fmax) + bsig.Min, Count: int64(fcount), }, nil } @@ -685,7 +676,7 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p // executeRangeSlice executes a range() call for a local slice. func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { - // Handle field ranges differently. + // Handle bsiGroup ranges differently. if c.HasConditionArg() { return e.executeBSIGroupRangeSlice(ctx, index, c, slice) } @@ -756,38 +747,29 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return row, nil } -// executeBSIGroupRangeSlice executes a range(field) call for a local slice. +// executeBSIGroupRangeSlice executes a range(bsiGroup) call for a local slice. func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { - // Parse frame, use default if unset. - frame, _ := c.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame - } - f := e.Holder.Frame(index, frame) - if f == nil { - return nil, ErrFrameNotFound - } - - // Remove frame field. - args := pql.CopyArgs(c.Args) - delete(args, "frame") - - // Only one conditional field should remain. - if len(args) == 0 { + // Only one conditional should be present. + if len(c.Args) == 0 { return nil, errors.New("Range(): condition required") - } else if len(args) > 1 { + } else if len(c.Args) > 1 { return nil, errors.New("Range(): too many arguments") } - // Extract condition field. - var fieldName string + // Extract conditional. + var frameName string var cond *pql.Condition - for k, v := range args { + for k, v := range c.Args { vv, ok := v.(*pql.Condition) if !ok { return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v) } - fieldName, cond = k, vv + frameName, cond = k, vv + } + + f := e.Holder.Frame(index, frameName) + if f == nil { + return nil, ErrFrameNotFound } // EQ null (not implemented: flip frag.NotNull with max ColumnID) @@ -799,19 +781,19 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, // Handle `!= null`. if cond.Op == pql.NEQ && cond.Value == nil { - // Find field. - field := f.bsiGroup(fieldName) - if field == nil { + // Find bsiGroup. + bsig := f.bsiGroup(frameName) + if bsig == nil { return nil, ErrBSIGroupNotFound } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, viewBSIGroupPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) if frag == nil { return NewRow(), nil } - return frag.NotNull(field.BitDepth()) + return frag.NotNull(bsig.BitDepth()) } else if cond.Op == pql.BETWEEN { @@ -826,33 +808,33 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // The reason we don't just call: - // return f.RangeBetween(fieldName, predicates[0], predicates[1]) + // return f.RangeBetween(frameName, predicates[0], predicates[1]) // here is because we need the call to be slice-specific. - // Find field. - field := f.bsiGroup(fieldName) - if field == nil { + // Find bsiGroup. + bsig := f.bsiGroup(frameName) + if bsig == nil { return nil, ErrBSIGroupNotFound } - baseValueMin, baseValueMax, outOfRange := field.baseValueBetween(predicates[0], predicates[1]) + baseValueMin, baseValueMax, outOfRange := bsig.baseValueBetween(predicates[0], predicates[1]) if outOfRange { return NewRow(), nil } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, viewBSIGroupPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) if frag == nil { return NewRow(), nil } // If the query is asking for the entire valid range, just return - // the not-null bitmap for the field. - if predicates[0] <= field.Min && predicates[1] >= field.Max { - return frag.NotNull(field.BitDepth()) + // the not-null bitmap for the bsiGroup. + if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max { + return frag.NotNull(bsig.BitDepth()) } - return frag.RangeBetween(field.BitDepth(), baseValueMin, baseValueMax) + return frag.RangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) } else { @@ -862,36 +844,36 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, return nil, errors.New("Range(): conditions only support integer values") } - // Find field. - field := f.bsiGroup(fieldName) - if field == nil { + // Find bsiGroup. + bsig := f.bsiGroup(frameName) + if bsig == nil { return nil, ErrBSIGroupNotFound } - baseValue, outOfRange := field.baseValue(cond.Op, value) + baseValue, outOfRange := bsig.baseValue(cond.Op, value) if outOfRange && cond.Op != pql.NEQ { return NewRow(), nil } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, viewBSIGroupPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) if frag == nil { return NewRow(), nil } - // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid field range. - if (cond.Op == pql.LT && value > field.Max) || (cond.Op == pql.LTE && value >= field.Max) || - (cond.Op == pql.GT && value < field.Min) || (cond.Op == pql.GTE && value <= field.Min) { - return frag.NotNull(field.BitDepth()) + // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. + if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || + (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { + return frag.NotNull(bsig.BitDepth()) } // outOfRange for NEQ should return all not-null. if outOfRange && cond.Op == pql.NEQ { - return frag.NotNull(field.BitDepth()) + return frag.NotNull(bsig.BitDepth()) } - f.Stats.Count("range:field", 1, 1.0) - return frag.RangeOp(cond.Op, field.BitDepth(), baseValue) + f.Stats.Count("range:bsigroup", 1, 1.0) + return frag.RangeOp(cond.Op, bsig.BitDepth(), baseValue) } } diff --git a/executor_test.go b/executor_test.go index d06a5027c..f042d3f8f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -834,7 +834,7 @@ func TestExecutor_Execute_Range(t *testing.T) { } t.Run("EQ", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo == 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -843,19 +843,19 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Run("NEQ", func(t *testing.T) { // NEQ null - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, other != null)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other != null)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo != 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo != 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, other != -20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other != -20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -864,7 +864,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("LT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo < 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo < 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -872,7 +872,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("LTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo <= 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo <= 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -880,7 +880,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("GT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo > 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo > 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -888,7 +888,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("GTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo >= 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo >= 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -896,7 +896,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("BETWEEN", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, other >< [1, 1000])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [1, 1000])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -905,7 +905,7 @@ func TestExecutor_Execute_Range(t *testing.T) { // Ensure that the NotNull code path gets run. t.Run("NotNull", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, other >< [0, 1000])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [0, 1000])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -913,7 +913,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("BelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo == 0)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 0)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -921,7 +921,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("AboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, foo == 200)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 200)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -929,7 +929,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("LTAboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, edge < 200)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(edge < 200)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) @@ -937,7 +937,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, edge > -200)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(edge > -200)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) @@ -945,13 +945,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("ErrFrameNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=bad_frame, foo >= 20)`), nil, nil); err != pilosa.ErrFrameNotFound { - t.Fatal(err) - } - }) - - t.Run("ErrBSIGroupNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=foo, bad_field >= 20)`), nil, nil); err != pilosa.ErrBSIGroupNotFound { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(bad_frame >= 20)`), nil, nil); err != pilosa.ErrFrameNotFound { t.Fatal(err) } }) From 0d44e586cc8da3081091fa851be8cf8e5a2651ef Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 16:14:26 -0500 Subject: [PATCH 32/78] remove field flag from pilosa import command --- cmd/import.go | 1 - ctl/import.go | 27 ++++++++++++++++++++------- ctl/import_test.go | 16 +++++++++++----- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index 8dd31181c..ea489f59d 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -55,7 +55,6 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.") - flags.StringVarP(&Importer.Field, "field", "", "", "Field to import into.") flags.BoolVar(&Importer.StringKeys, "string-keys", false, "Treat payload as string keys.") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") diff --git a/ctl/import.go b/ctl/import.go index 4fdfba444..493e1429b 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -46,9 +46,6 @@ type ImportCommand struct { // CreateSchema ensures the schema exists before import CreateSchema bool - // For Range-Encoded fields, name of the Field to import into. - Field string `json:"field"` - // Indicates that the payload should be treated as string keys. StringKeys bool `json:"StringKeys"` @@ -105,10 +102,26 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } } + // Determine the frame type in order to correctly handle the input data. + frameType := pilosa.DefaultFrameType + schema, err := cmd.Client.Schema(ctx) + if err != nil { + return errors.Wrap(err, "getting schema") + } + for _, index := range schema { + if index.Name == cmd.Index { + for _, frame := range index.Frames { + if frame.Name == cmd.Frame { + frameType = frame.Options.Type + } + } + } + } + // Import each path and import by slice. for _, path := range cmd.Paths { logger.Printf("parsing: %s", path) - if err := cmd.importPath(ctx, path); err != nil { + if err := cmd.importPath(ctx, frameType, path); err != nil { return err } } @@ -129,9 +142,9 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { } // importPath parses a path into bits and imports it to the server. -func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { - // If a field is provided, treat the import data as values to be range-encoded. - if cmd.Field != "" { +func (cmd *ImportCommand) importPath(ctx context.Context, frameType, path string) error { + // If frameType is `int`, treat the import data as values to be range-encoded. + if frameType == pilosa.FrameTypeInt { return cmd.bufferFieldValues(ctx, path) } else { if cmd.StringKeys { diff --git a/ctl/import_test.go b/ctl/import_test.go index 7c6c02721..7c64944f6 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -82,9 +82,7 @@ func TestImportCommand_Run(t *testing.T) { } } -// TODO: revisit this test once Frame is renamed Field -// Ensure that the ImportValue path runs (note: we have specified a value -// for cm.Field.) +// Ensure that the ImportValue path runs. func TestImportCommand_RunValue(t *testing.T) { buf := bytes.Buffer{} @@ -112,7 +110,6 @@ func TestImportCommand_RunValue(t *testing.T) { cm.Index = "i" cm.Frame = "f" - cm.Field = "f" cm.Paths = []string{file.Name()} err = cm.Run(ctx) if err != nil { @@ -122,10 +119,19 @@ func TestImportCommand_RunValue(t *testing.T) { func TestImportCommand_InvalidFile(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + s := test.NewServer() + defer s.Close() + + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder + buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - cm.Host = "anyhost" + cm.Host = s.Host() cm.Index = "i" cm.Frame = "f" file, err := ioutil.TempFile("", "import.csv") From 96208283df9f749a90d1e7c41ca624102885b7bf Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 16:38:29 -0500 Subject: [PATCH 33/78] remove final instances of Field --- ctl/import.go | 14 +++++++------- fragment_test.go | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ctl/import.go b/ctl/import.go index 493e1429b..c6c0f0188 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -145,7 +145,7 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { func (cmd *ImportCommand) importPath(ctx context.Context, frameType, path string) error { // If frameType is `int`, treat the import data as values to be range-encoded. if frameType == pilosa.FrameTypeInt { - return cmd.bufferFieldValues(ctx, path) + return cmd.bufferValues(ctx, path) } else { if cmd.StringKeys { return cmd.bufferBitsK(ctx, path) @@ -358,8 +358,8 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er return nil } -// bufferFieldValues buffers slices of fieldValues to be imported as a batch. -func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) error { +// bufferValues buffers slices of fieldValues to be imported as a batch. +func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error { a := make([]pilosa.FieldValue, 0, cmd.BufferSize) var r *csv.Reader @@ -418,7 +418,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er // If we've reached the buffer size then import field values. if len(a) == cmd.BufferSize { - if err := cmd.importFieldValues(ctx, a); err != nil { + if err := cmd.importValues(ctx, a); err != nil { return err } a = a[:0] @@ -426,15 +426,15 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er } // If there are still values in the buffer then flush them. - if err := cmd.importFieldValues(ctx, a); err != nil { + if err := cmd.importValues(ctx, a); err != nil { return err } return nil } -// importFieldValues sends batches of fieldValues to the server. -func (cmd *ImportCommand) importFieldValues(ctx context.Context, vals []pilosa.FieldValue) error { +// importValues sends batches of fieldValues to the server. +func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldValue) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // Group vals by slice. diff --git a/fragment_test.go b/fragment_test.go index e1c9b7fd4..6a96fb677 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -220,7 +220,7 @@ func TestFragment_SetValue(t *testing.T) { } // Ensure a fragment can sum field values. -func TestFragment_FieldSum(t *testing.T) { +func TestFragment_Sum(t *testing.T) { const bitDepth = 16 f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") @@ -259,7 +259,7 @@ func TestFragment_FieldSum(t *testing.T) { } // Ensure a fragment can find the min and max of field values. -func TestFragment_FieldMinMax(t *testing.T) { +func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") @@ -332,7 +332,7 @@ func TestFragment_FieldMinMax(t *testing.T) { } // Ensure a fragment query for matching fields. -func TestFragment_FieldRange(t *testing.T) { +func TestFragment_Range(t *testing.T) { const bitDepth = 16 t.Run("EQ", func(t *testing.T) { From 80d656ae9c9b93c4a53996d5215190ea8bac9a82 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 16:56:16 -0500 Subject: [PATCH 34/78] final removal of field instances --- ctl/import.go | 10 +++++----- executor_test.go | 20 ++++++++++---------- fragment_test.go | 32 ++++++++++++++++---------------- frame_test.go | 4 ++-- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/ctl/import.go b/ctl/import.go index c6c0f0188..45bf81d42 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -358,7 +358,7 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er return nil } -// bufferValues buffers slices of fieldValues to be imported as a batch. +// bufferValues buffers slices of FieldValues to be imported as a batch. func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error { a := make([]pilosa.FieldValue, 0, cmd.BufferSize) @@ -407,7 +407,7 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error { } val.ColumnID = columnID - // Parse field value. + // Parse FieldValue. value, err := strconv.ParseInt(record[1], 10, 64) if err != nil { return fmt.Errorf("invalid value on row %d: %q", rnum, record[1]) @@ -416,7 +416,7 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error { a = append(a, val) - // If we've reached the buffer size then import field values. + // If we've reached the buffer size then import FieldValues. if len(a) == cmd.BufferSize { if err := cmd.importValues(ctx, a); err != nil { return err @@ -433,7 +433,7 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error { return nil } -// importValues sends batches of fieldValues to the server. +// importValues sends batches of FieldValues to the server. func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldValue) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) @@ -441,7 +441,7 @@ func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldV logger.Printf("grouping %d vals", len(vals)) valsBySlice := pilosa.FieldValues(vals).GroupBySlice() - // Parse path into field values. + // Parse path into FieldValues. for slice, vals := range valsBySlice { if cmd.Sort { sort.Sort(pilosa.FieldValues(vals)) diff --git a/executor_test.go b/executor_test.go index f042d3f8f..89e54a1de 100644 --- a/executor_test.go +++ b/executor_test.go @@ -281,7 +281,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Fatal(err) } - // Set field values. + // Set bsiGroup values. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f=25)`), nil, nil); err != nil { t.Fatal(err) @@ -355,8 +355,8 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { t.Fatal(err) } - // Set two fields on f/10. - // Also set fields on other bitmaps and frames to test isolation. + // Set two attrs on f/10. + // Also set attrs on other bitmaps and frames to test isolation. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) @@ -617,9 +617,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) { for i, tt := range tests { var pql string if tt.filter == "" { - pql = `Min(frame=f, field=f)` + pql = `Min(frame=f)` } else { - pql = fmt.Sprintf(`Min(%s, frame=f, field=f)`, tt.filter) + pql = fmt.Sprintf(`Min(%s, frame=f)`, tt.filter) } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) @@ -643,9 +643,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) { for i, tt := range tests { var pql string if tt.filter == "" { - pql = `Max(frame=f, field=f)` + pql = `Max(frame=f)` } else { - pql = fmt.Sprintf(`Max(%s, frame=f, field=f)`, tt.filter) + pql = fmt.Sprintf(`Max(%s, frame=f)`, tt.filter) } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) @@ -711,7 +711,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { } t.Run("NoFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=foo, field=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=foo)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -719,7 +719,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=x, row=0), frame=foo, field=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=x, row=0), frame=foo)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -769,7 +769,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { }) } -// Ensure a Range(field) query can be executed. +// Ensure a Range(bsiGroup) query can be executed. func TestExecutor_Execute_Range(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() diff --git a/fragment_test.go b/fragment_test.go index 6a96fb677..b72450fbb 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -96,7 +96,7 @@ func TestFragment_ClearBit(t *testing.T) { } } -// Ensure a fragment can set & read a field value. +// Ensure a fragment can set & read a value. func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") @@ -219,7 +219,7 @@ func TestFragment_SetValue(t *testing.T) { }) } -// Ensure a fragment can sum field values. +// Ensure a fragment can sum values. func TestFragment_Sum(t *testing.T) { const bitDepth = 16 @@ -258,7 +258,7 @@ func TestFragment_Sum(t *testing.T) { }) } -// Ensure a fragment can find the min and max of field values. +// Ensure a fragment can find the min and max of values. func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 @@ -331,7 +331,7 @@ func TestFragment_MinMax(t *testing.T) { }) } -// Ensure a fragment query for matching fields. +// Ensure a fragment query for matching values. func TestFragment_Range(t *testing.T) { const bitDepth = 16 @@ -400,28 +400,28 @@ func TestFragment_Range(t *testing.T) { t.Fatal(err) } - // Query for fields less than (ending with set column). + // Query for values less than (ending with set column). if b, err := f.RangeOp(pql.LT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than (ending with unset column). + // Query for values less than (ending with unset column). if b, err := f.RangeOp(pql.LT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than or equal to (ending with set column). + // Query for values less than or equal to (ending with set column). if b, err := f.RangeOp(pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than or equal to (ending with unset column). + // Query for values less than or equal to (ending with unset column). if b, err := f.RangeOp(pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { @@ -448,28 +448,28 @@ func TestFragment_Range(t *testing.T) { t.Fatal(err) } - // Query for fields greater than (ending with unset bit). + // Query for values greater than (ending with unset bit). if b, err := f.RangeOp(pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than (ending with set bit). + // Query for values greater than (ending with set bit). if b, err := f.RangeOp(pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with unset bit). + // Query for values greater than or equal to (ending with unset bit). if b, err := f.RangeOp(pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with set bit). + // Query for values greater than or equal to (ending with set bit). if b, err := f.RangeOp(pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { @@ -496,28 +496,28 @@ func TestFragment_Range(t *testing.T) { t.Fatal(err) } - // Query for fields greater than (ending with unset column). + // Query for values greater than (ending with unset column). if b, err := f.RangeBetween(bitDepth, 300, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than (ending with set column). + // Query for values greater than (ending with set column). if b, err := f.RangeBetween(bitDepth, 301, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with unset column). + // Query for values greater than or equal to (ending with unset column). if b, err := f.RangeBetween(bitDepth, 301, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with set column). + // Query for values greater than or equal to (ending with set column). if b, err := f.RangeBetween(bitDepth, 300, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) { diff --git a/frame_test.go b/frame_test.go index 739b34ed8..3964af785 100644 --- a/frame_test.go +++ b/frame_test.go @@ -71,7 +71,7 @@ func TestFrame_SetTimeQuantum(t *testing.T) { } } -// Ensure a frame can set & read a field value. +// Ensure a frame can set & read a bsiGroup value. func TestFrame_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { idx := test.MustOpenIndex() @@ -86,7 +86,7 @@ func TestFrame_SetValue(t *testing.T) { t.Fatal(err) } - // Set value on field. + // Set value on frame. if changed, err := f.SetValue(100, 21); err != nil { t.Fatal(err) } else if !changed { From 1d3c4d6fcb50ffb1364dd2712d6df578d9b35313 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 17:52:22 -0500 Subject: [PATCH 35/78] first pass at GoRename Frame to Field in frame.go --- api.go | 4 +- client.go | 8 +- client_test.go | 4 +- cluster_internal_test.go | 2 +- cluster_test.go | 2 +- ctl/import.go | 6 +- diagnostics.go | 2 +- executor.go | 4 +- executor_test.go | 70 +++++++------- fragment_test.go | 4 +- frame.go | 194 +++++++++++++++++++-------------------- frame_test.go | 30 +++--- handler.go | 4 +- handler_internal_test.go | 4 +- handler_test.go | 16 ++-- holder.go | 8 +- holder_test.go | 16 ++-- index.go | 30 +++--- index_test.go | 22 ++--- server.go | 2 +- server/cluster_test.go | 10 +- server/server_test.go | 12 +-- server_test.go | 2 +- stats_test.go | 2 +- test/cluster.go | 2 +- test/frame.go | 16 ++-- test/holder.go | 6 +- test/index.go | 8 +- utils_test.go | 2 +- 29 files changed, 246 insertions(+), 246 deletions(-) diff --git a/api.go b/api.go index e528a4b21..fec6b3129 100644 --- a/api.go +++ b/api.go @@ -218,7 +218,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { } // CreateFrame makes the named frame in the named index with the given options. -func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { +func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FieldOptions) (*Field, error) { if err := api.validate(apiCreateFrame); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -679,7 +679,7 @@ func (api *API) LongQueryTime() time.Duration { return api.Cluster.LongQueryTime } -func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { +func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Field, error) { // Validate that this handler owns the slice. if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) diff --git a/client.go b/client.go index 1d78dde23..4c8ba2157 100644 --- a/client.go +++ b/client.go @@ -330,7 +330,7 @@ func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, optio return err } -func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error { +func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, frameName string, options FieldOptions) error { err := c.CreateFrame(ctx, indexName, frameName, options) if err == nil || err == ErrFrameExists { return nil @@ -616,7 +616,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame s } // CreateFrame creates a new frame on the server. -func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error { +func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FieldOptions) error { if index == "" { return ErrIndexRequired } @@ -1056,10 +1056,10 @@ type InternalClient interface { Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error ImportK(ctx context.Context, index, frame string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error - EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error + EnsureFrame(ctx context.Context, indexName string, frameName string, options FieldOptions) error ImportValue(ctx context.Context, index, frame string, slice uint64, vals []FieldValue) error ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error - CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error + CreateFrame(ctx context.Context, index, frame string, opt FieldOptions) error FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) diff --git a/client_test.go b/client_test.go index 1edd37bc2..6f1931e29 100644 --- a/client_test.go +++ b/client_test.go @@ -246,8 +246,8 @@ func TestClient_ImportValue(t *testing.T) { fldName := "f" - fo := pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + fo := pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: -100, Max: 100, } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 6b1f6e72b..acccd37d3 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -145,7 +145,7 @@ func TestFragSources(t *testing.T) { c5.addNodeBasicSorted(node3) idx := newIndexWithTempPath("i") - frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{}) + frame, err := idx.CreateFrameIfNotExists("f", FieldOptions{}) if err != nil { t.Fatal(err) } diff --git a/cluster_test.go b/cluster_test.go index 99b322a0e..f8903aafd 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -416,7 +416,7 @@ func TestCluster_ResizeStates(t *testing.T) { } // Add Bit Data to node0. - if err := tc.CreateFrame("i", "f", FrameOptions{}); err != nil { + if err := tc.CreateFrame("i", "f", FieldOptions{}); err != nil { t.Fatal(err) } tc.SetBit("i", "f", "standard", 1, 101, nil) diff --git a/ctl/import.go b/ctl/import.go index 45bf81d42..3a7835cb2 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -41,7 +41,7 @@ type ImportCommand struct { // Options for index & frame to be created if they don't exist IndexOptions pilosa.IndexOptions - FrameOptions pilosa.FrameOptions + FrameOptions pilosa.FieldOptions // CreateSchema ensures the schema exists before import CreateSchema bool @@ -103,7 +103,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } // Determine the frame type in order to correctly handle the input data. - frameType := pilosa.DefaultFrameType + frameType := pilosa.DefaultFieldType schema, err := cmd.Client.Schema(ctx) if err != nil { return errors.Wrap(err, "getting schema") @@ -144,7 +144,7 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { // importPath parses a path into bits and imports it to the server. func (cmd *ImportCommand) importPath(ctx context.Context, frameType, path string) error { // If frameType is `int`, treat the import data as values to be range-encoded. - if frameType == pilosa.FrameTypeInt { + if frameType == pilosa.FieldTypeInt { return cmd.bufferValues(ctx, path) } else { if cmd.StringKeys { diff --git a/diagnostics.go b/diagnostics.go index 1968a4e74..10ecfa22d 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -227,7 +227,7 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { numIndexes += 1 for _, frame := range index.Frames() { numFrames += 1 - if frame.Type() == FrameTypeInt { + if frame.Type() == FieldTypeInt { bsiFieldCount += 1 } if frame.TimeQuantum() != "" { diff --git a/executor.go b/executor.go index eee4fa0ee..fd98f7189 100644 --- a/executor.go +++ b/executor.go @@ -983,7 +983,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal } // executeClearBitView executes a ClearBit() call for a single view. -func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { +func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Field, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false for _, node := range e.Cluster.SliceNodes(index, slice) { @@ -1058,7 +1058,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, } // executeSetBitView executes a SetBit() call for a specific view. -func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) { +func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.Call, f *Field, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false diff --git a/executor_test.go b/executor_test.go index 89e54a1de..0d70ae37f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -33,7 +33,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := index.CreateFrame("f", pilosa.FrameOptions{}) + f, err := index.CreateFrame("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } @@ -83,7 +83,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if _, err := index.CreateFrame("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -271,13 +271,13 @@ func TestExecutor_Execute_SetValue(t *testing.T) { // Create frames. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + if _, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 0, Max: 50, }); err != nil { t.Fatal(err) - } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil { + } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -311,8 +311,8 @@ func TestExecutor_Execute_SetValue(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + if _, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 0, Max: 100, }); err != nil { @@ -349,9 +349,9 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Create frames. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + if _, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil { + } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -388,9 +388,9 @@ func TestExecutor_Execute_TopN(t *testing.T) { // Set columns for rows 0, 10, & 20 across two slices. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil { + } else if _, err := idx.CreateFrame("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{}); err != nil { + } else if _, err := idx.CreateFrame("other", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` SetBit(frame=f, row=0, col=0) @@ -572,12 +572,12 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("x", pilosa.FrameOptions{}); err != nil { + if _, err := idx.CreateFrame("x", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + if _, err := idx.CreateFrame("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: -10, Max: 100, }); err != nil { @@ -667,28 +667,28 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("x", pilosa.FrameOptions{}); err != nil { + if _, err := idx.CreateFrame("x", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("foo", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + if _, err := idx.CreateFrame("foo", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 10, Max: 100, }); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + if _, err := idx.CreateFrame("bar", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 0, Max: 100000, }); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + if _, err := idx.CreateFrame("other", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 0, Max: 1000, }); err != nil { @@ -737,8 +737,8 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) // Create frame. - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeTime, + if _, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeTime, TimeQuantum: pilosa.TimeQuantum("YMDH"), }); err != nil { t.Fatal(err) @@ -780,36 +780,36 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if _, err := idx.CreateFrame("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("foo", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + if _, err := idx.CreateFrame("foo", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 10, Max: 100, }); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + if _, err := idx.CreateFrame("bar", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 0, Max: 100000, }); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + if _, err := idx.CreateFrame("other", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 0, Max: 1000, }); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("edge", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + if _, err := idx.CreateFrame("edge", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: -100, Max: 100, }); err != nil { @@ -1068,7 +1068,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Create frame. - if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -1120,7 +1120,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Create frame. - if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := f.SetTimeQuantum("Y"); err != nil { t.Fatal(err) @@ -1223,7 +1223,7 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - index.CreateFrame("f", pilosa.FrameOptions{}) + index.CreateFrame("f", pilosa.FieldOptions{}) targetAttrs := map[string]interface{}{ "foo": "bar", } diff --git a/fragment_test.go b/fragment_test.go index b72450fbb..2d618aa81 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -754,7 +754,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { defer index.Close() // Create frame. - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) + frame, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) if err != nil { t.Fatal(err) } @@ -924,7 +924,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { defer index.Close() // Create frame. - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked}) + frame, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) if err != nil { t.Fatal(err) } diff --git a/frame.go b/frame.go index ac9eb5ee7..b69848344 100644 --- a/frame.go +++ b/frame.go @@ -31,7 +31,7 @@ import ( // Default frame settings. const ( - DefaultFrameType = FrameTypeSet + DefaultFieldType = FieldTypeSet DefaultCacheType = CacheTypeRanked @@ -39,15 +39,15 @@ const ( DefaultCacheSize = 50000 ) -// Frame types. +// Field types. const ( - FrameTypeSet = "set" - FrameTypeInt = "int" - FrameTypeTime = "time" + FieldTypeSet = "set" + FieldTypeInt = "int" + FieldTypeTime = "time" ) -// Frame represents a container for views. -type Frame struct { +// Field represents a container for views. +type Field struct { mu sync.RWMutex path string index string @@ -61,33 +61,33 @@ type Frame struct { broadcaster Broadcaster Stats StatsClient - // Frame options. - options FrameOptions + // Field options. + options FieldOptions bsiGroups []*bsiGroup Logger Logger } -// FrameOption is a functional option type for pilosa.Frame. -type FrameOption func(f *Frame) error +// FieldOption is a functional option type for pilosa.Fielde. +type FieldOption func(f *Field) error -// TODO: break these out into separate Options (not a FrameOptions object) -func OptFrameFrameOptions(o FrameOptions) FrameOption { - return func(f *Frame) error { +// TODO: break these out into separate Options (not a FieldOptions object) +func OptFieldFieldOptions(o FieldOptions) FieldOption { + return func(f *Field) error { f.options = o return nil } } -// NewFrame returns a new instance of frame. -func NewFrame(path, index, name string, opts ...FrameOption) (*Frame, error) { +// NewField returns a new instance of frame. +func NewField(path, index, name string, opts ...FieldOption) (*Field, error) { err := ValidateName(name) if err != nil { return nil, err } - f := &Frame{ + f := &Field{ path: path, index: index, name: name, @@ -99,8 +99,8 @@ func NewFrame(path, index, name string, opts ...FrameOption) (*Frame, error) { broadcaster: NopBroadcaster, Stats: NopStatsClient, - options: FrameOptions{ - Type: DefaultFrameType, + options: FieldOptions{ + Type: DefaultFieldType, CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, }, @@ -119,19 +119,19 @@ func NewFrame(path, index, name string, opts ...FrameOption) (*Frame, error) { } // Name returns the name the frame was initialized with. -func (f *Frame) Name() string { return f.name } +func (f *Field) Name() string { return f.name } // Index returns the index name the frame was initialized with. -func (f *Frame) Index() string { return f.index } +func (f *Field) Index() string { return f.index } // Path returns the path the frame was initialized with. -func (f *Frame) Path() string { return f.path } +func (f *Field) Path() string { return f.path } // RowAttrStore returns the attribute storage. -func (f *Frame) RowAttrStore() AttrStore { return f.rowAttrStore } +func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore } // MaxSlice returns the max slice in the frame. -func (f *Frame) MaxSlice() uint64 { +func (f *Field) MaxSlice() uint64 { f.mu.RLock() defer f.mu.RUnlock() @@ -145,14 +145,14 @@ func (f *Frame) MaxSlice() uint64 { } // Type returns the frame type. -func (f *Frame) Type() string { +func (f *Field) Type() string { f.mu.RLock() defer f.mu.RUnlock() return f.options.Type } // CacheType returns the caching mode for the frame. -func (f *Frame) CacheType() string { +func (f *Field) CacheType() string { f.mu.RLock() defer f.mu.RUnlock() return f.options.CacheType @@ -160,7 +160,7 @@ func (f *Frame) CacheType() string { // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 -func (f *Frame) SetCacheSize(v uint32) error { +func (f *Field) SetCacheSize(v uint32) error { f.mu.Lock() defer f.mu.Unlock() @@ -179,7 +179,7 @@ func (f *Frame) SetCacheSize(v uint32) error { } // CacheSize returns the ranked frame cache size. -func (f *Frame) CacheSize() uint32 { +func (f *Field) CacheSize() uint32 { f.mu.RLock() v := f.options.CacheSize f.mu.RUnlock() @@ -187,14 +187,14 @@ func (f *Frame) CacheSize() uint32 { } // Options returns all options for this frame. -func (f *Frame) Options() FrameOptions { +func (f *Field) Options() FieldOptions { f.mu.RLock() defer f.mu.RUnlock() return f.options } // Open opens and initializes the frame. -func (f *Frame) Open() error { +func (f *Field) Open() error { if err := func() error { // Ensure the frame's path exists. if err := os.MkdirAll(f.path, 0777); err != nil { @@ -228,7 +228,7 @@ func (f *Frame) Open() error { } // openViews opens and initializes the views inside the frame. -func (f *Frame) openViews() error { +func (f *Field) openViews() error { file, err := os.Open(filepath.Join(f.path, "views")) if os.IsNotExist(err) { return nil @@ -260,7 +260,7 @@ func (f *Frame) openViews() error { } // loadMeta reads meta data for the frame, if any. -func (f *Frame) loadMeta() error { +func (f *Field) loadMeta() error { var pb internal.FrameMeta // Read data from meta file. @@ -287,7 +287,7 @@ func (f *Frame) loadMeta() error { } // saveMeta writes meta data for the frame. -func (f *Frame) saveMeta() error { +func (f *Field) saveMeta() error { // Marshal metadata. fo := f.options buf, err := proto.Marshal(fo.Encode()) @@ -304,10 +304,10 @@ func (f *Frame) saveMeta() error { } // applyOptions configures the frame based on opt. -func (f *Frame) applyOptions(opt FrameOptions) error { +func (f *Field) applyOptions(opt FieldOptions) error { switch opt.Type { - case FrameTypeSet, "": - f.options.Type = FrameTypeSet + case FieldTypeSet, "": + f.options.Type = FieldTypeSet if opt.CacheType != "" { f.options.CacheType = opt.CacheType } @@ -317,7 +317,7 @@ func (f *Frame) applyOptions(opt FrameOptions) error { f.options.Min = 0 f.options.Max = 0 f.options.TimeQuantum = "" - case FrameTypeInt: + case FieldTypeInt: f.options.Type = opt.Type f.options.CacheType = CacheTypeNone f.options.CacheSize = 0 @@ -339,7 +339,7 @@ func (f *Frame) applyOptions(opt FrameOptions) error { if err := f.createBSIGroup(bsig); err != nil { return errors.Wrap(err, "creating bsigroup") } - case FrameTypeTime: + case FieldTypeTime: f.options.Type = opt.Type f.options.CacheType = CacheTypeNone f.options.CacheSize = 0 @@ -358,7 +358,7 @@ func (f *Frame) applyOptions(opt FrameOptions) error { } // Close closes the frame and its views. -func (f *Frame) Close() error { +func (f *Field) Close() error { f.mu.Lock() defer f.mu.Unlock() @@ -379,7 +379,7 @@ func (f *Frame) Close() error { } // bsiGroup returns a bsiGroup by name. -func (f *Frame) bsiGroup(name string) *bsiGroup { +func (f *Field) bsiGroup(name string) *bsiGroup { f.mu.RLock() defer f.mu.RUnlock() for _, bsig := range f.bsiGroups { @@ -391,7 +391,7 @@ func (f *Frame) bsiGroup(name string) *bsiGroup { } // hasBSIGroup returns true if a bsiGroup exists on the frame. -func (f *Frame) hasBSIGroup(name string) bool { +func (f *Field) hasBSIGroup(name string) bool { for _, bsig := range f.bsiGroups { if bsig.Name == name { return true @@ -401,7 +401,7 @@ func (f *Frame) hasBSIGroup(name string) bool { } // createBSIGroup creates a new bsiGroup on the frame. -func (f *Frame) createBSIGroup(bsig *bsiGroup) error { +func (f *Field) createBSIGroup(bsig *bsiGroup) error { f.mu.Lock() defer f.mu.Unlock() @@ -414,7 +414,7 @@ func (f *Frame) createBSIGroup(bsig *bsiGroup) error { } // addBSIGroup adds a single bsiGroup to bsiGroups. -func (f *Frame) addBSIGroup(bsig *bsiGroup) error { +func (f *Field) addBSIGroup(bsig *bsiGroup) error { if err := bsig.validate(); err != nil { return errors.Wrap(err, "validating bsigroup") } else if f.hasBSIGroup(bsig.Name) { @@ -433,7 +433,7 @@ func (f *Frame) addBSIGroup(bsig *bsiGroup) error { } // deleteBSIGroupAndView deletes an existing bsiGroup on the schema. -func (f *Frame) deleteBSIGroupAndView(name string) error { +func (f *Field) deleteBSIGroupAndView(name string) error { f.mu.Lock() defer f.mu.Unlock() @@ -458,7 +458,7 @@ func (f *Frame) deleteBSIGroupAndView(name string) error { } // deleteBSIGroup removes a single bsiGroup from bsiGroups. -func (f *Frame) deleteBSIGroup(name string) error { +func (f *Field) deleteBSIGroup(name string) error { for i, bsig := range f.bsiGroups { if bsig.Name == name { copy(f.bsiGroups[i:], f.bsiGroups[i+1:]) @@ -470,14 +470,14 @@ func (f *Frame) deleteBSIGroup(name string) error { } // TimeQuantum returns the time quantum for the frame. -func (f *Frame) TimeQuantum() TimeQuantum { +func (f *Field) TimeQuantum() TimeQuantum { f.mu.Lock() defer f.mu.Unlock() return f.options.TimeQuantum } // SetTimeQuantum sets the time quantum for the frame. -func (f *Frame) SetTimeQuantum(q TimeQuantum) error { +func (f *Field) SetTimeQuantum(q TimeQuantum) error { f.mu.Lock() defer f.mu.Unlock() @@ -498,21 +498,21 @@ func (f *Frame) SetTimeQuantum(q TimeQuantum) error { } // ViewPath returns the path to a view in the frame. -func (f *Frame) ViewPath(name string) string { +func (f *Field) ViewPath(name string) string { return filepath.Join(f.path, "views", name) } // View returns a view in the frame by name. -func (f *Frame) View(name string) *View { +func (f *Field) View(name string) *View { f.mu.RLock() defer f.mu.RUnlock() return f.view(name) } -func (f *Frame) view(name string) *View { return f.views[name] } +func (f *Field) view(name string) *View { return f.views[name] } // Views returns a list of all views in the frame. -func (f *Frame) Views() []*View { +func (f *Field) Views() []*View { f.mu.RLock() defer f.mu.RUnlock() @@ -524,7 +524,7 @@ func (f *Frame) Views() []*View { } // viewNames returns a list of all views (as a string) in the frame. -func (f *Frame) viewNames() []string { +func (f *Field) viewNames() []string { f.mu.Lock() defer f.mu.Unlock() @@ -536,7 +536,7 @@ func (f *Frame) viewNames() []string { } // RecalculateCaches recalculates caches on every view in the frame. -func (f *Frame) RecalculateCaches() { +func (f *Field) RecalculateCaches() { for _, view := range f.Views() { view.RecalculateCaches() } @@ -544,7 +544,7 @@ func (f *Frame) RecalculateCaches() { // CreateViewIfNotExists returns the named view, creating it if necessary. // Additionally, a CreateViewMessage is sent to the cluster. -func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { +func (f *Field) CreateViewIfNotExists(name string) (*View, error) { view, created, err := f.createViewIfNotExistsBase(name) if err != nil { @@ -569,7 +569,7 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { // createViewIfNotExistsBase returns the named view, creating it if necessary. // The returned bool indicates whether the view was created or not. -func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) { +func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) { f.mu.Lock() defer f.mu.Unlock() @@ -588,7 +588,7 @@ func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) { return view, true, nil } -func (f *Frame) newView(path, name string) *View { +func (f *Field) newView(path, name string) *View { view := NewView(path, f.index, f.name, name, f.options.CacheSize) view.cacheType = f.options.CacheType view.Logger = f.Logger @@ -599,7 +599,7 @@ func (f *Frame) newView(path, name string) *View { } // DeleteView removes the view from the frame. -func (f *Frame) DeleteView(name string) error { +func (f *Field) DeleteView(name string) error { view := f.views[name] if view == nil { return ErrInvalidView @@ -621,7 +621,7 @@ func (f *Frame) DeleteView(name string) error { } // SetBit sets a bit on a view within the frame. -func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { +func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. if !IsValidView(name) { return false, ErrInvalidView @@ -663,7 +663,7 @@ func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed } // ClearBit clears a bit within the frame. -func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { +func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. if !IsValidView(name) { return false, ErrInvalidView @@ -705,7 +705,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change } // Value reads a frame value for a column. -func (f *Frame) Value(columnID uint64) (value int64, exists bool, err error) { +func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { bsig := f.bsiGroup(f.name) if bsig == nil { return 0, false, ErrBSIGroupNotFound @@ -727,7 +727,7 @@ func (f *Frame) Value(columnID uint64) (value int64, exists bool, err error) { } // SetValue sets a frame value for a column. -func (f *Frame) SetValue(columnID uint64, value int64) (changed bool, err error) { +func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) { // Fetch bsiGroup and validate value. bsig := f.bsiGroup(f.name) if bsig == nil { @@ -752,7 +752,7 @@ func (f *Frame) SetValue(columnID uint64, value int64) (changed bool, err error) // Sum returns the sum and count for a frame. // An optional filtering row can be provided. -func (f *Frame) Sum(filter *Row, name string) (sum, count int64, err error) { +func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) { bsig := f.bsiGroup(name) if bsig == nil { return 0, 0, ErrBSIGroupNotFound @@ -772,7 +772,7 @@ func (f *Frame) Sum(filter *Row, name string) (sum, count int64, err error) { // Min returns the min for a frame. // An optional filtering row can be provided. -func (f *Frame) Min(filter *Row, name string) (min, count int64, err error) { +func (f *Field) Min(filter *Row, name string) (min, count int64, err error) { bsig := f.bsiGroup(name) if bsig == nil { return 0, 0, ErrBSIGroupNotFound @@ -792,7 +792,7 @@ func (f *Frame) Min(filter *Row, name string) (min, count int64, err error) { // Max returns the max for a frame. // An optional filtering row can be provided. -func (f *Frame) Max(filter *Row, name string) (max, count int64, err error) { +func (f *Field) Max(filter *Row, name string) (max, count int64, err error) { bsig := f.bsiGroup(name) if bsig == nil { return 0, 0, ErrBSIGroupNotFound @@ -810,7 +810,7 @@ func (f *Frame) Max(filter *Row, name string) (max, count int64, err error) { return int64(vmax) + bsig.Min, int64(vcount), nil } -func (f *Frame) Range(name string, op pql.Token, predicate int64) (*Row, error) { +func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) { // Retrieve and validate bsiGroup. bsig := f.bsiGroup(name) if bsig == nil { @@ -833,7 +833,7 @@ func (f *Frame) Range(name string, op pql.Token, predicate int64) (*Row, error) return view.rangeOp(op, bsig.BitDepth(), baseValue) } -func (f *Frame) RangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { +func (f *Field) RangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { // Retrieve and validate bsiGroup. bsig := f.bsiGroup(name) if bsig == nil { @@ -857,7 +857,7 @@ func (f *Frame) RangeBetween(name string, predicateMin, predicateMax int64) (*Ro } // Import bulk imports data. -func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error { +func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error { // Determine quantum if timestamps are set. q := f.TimeQuantum() if hasTime(timestamps) && q == "" { @@ -914,7 +914,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro } // ImportValue bulk imports range-encoded value data. -func (f *Frame) ImportValue(columnIDs []uint64, values []int64) error { +func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { viewName := viewBSIGroupPrefix + f.name // Get the bsiGroup so we know bitDepth. bsig := f.bsiGroup(f.name) @@ -970,17 +970,17 @@ func (f *Frame) ImportValue(columnIDs []uint64, values []int64) error { return nil } -// encodeFrames converts a into its internal representation. -func encodeFrames(a []*Frame) []*internal.Frame { +// encodeFields converts a into its internal representation. +func encodeFields(a []*Field) []*internal.Frame { other := make([]*internal.Frame, len(a)) for i := range a { - other[i] = encodeFrame(a[i]) + other[i] = encodeField(a[i]) } return other } -// encodeFrame converts f into its internal representation. -func encodeFrame(f *Frame) *internal.Frame { +// encodeField converts f into its internal representation. +func encodeField(f *Field) *internal.Frame { fo := f.options return &internal.Frame{ Name: f.name, @@ -989,45 +989,45 @@ func encodeFrame(f *Frame) *internal.Frame { } } -type frameSlice []*Frame +type fieldSlice []*Field -func (p frameSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p frameSlice) Len() int { return len(p) } -func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } +func (p fieldSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p fieldSlice) Len() int { return len(p) } +func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } -// FrameInfo represents schema information for a frame. -type FrameInfo struct { +// FieldInfo represents schema information for a frame. +type FieldInfo struct { Name string `json:"name"` - Options FrameOptions `json:"options"` + Options FieldOptions `json:"options"` Views []*ViewInfo `json:"views,omitempty"` } -type frameInfoSlice []*FrameInfo +type fieldInfoSlice []*FieldInfo -func (p frameInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p frameInfoSlice) Len() int { return len(p) } -func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } +func (p fieldInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p fieldInfoSlice) Len() int { return len(p) } +func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } -// FrameOptions represents options to set when initializing a frame. -type FrameOptions struct { +// FieldOptions represents options to set when initializing a field. +type FieldOptions struct { Type string `json:"type,omitempty"` CacheType string `json:"cacheType,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` Min int64 `json:"min,omitempty"` Max int64 `json:"max,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` // TODO: rename this Quantum? + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } -// Validate ensures that FrameOption values are valid. -func (o *FrameOptions) Validate() error { +// Validate ensures that FieldOption values are valid. +func (o *FieldOptions) Validate() error { switch o.Type { - case FrameTypeSet, "": + case FieldTypeSet, "": // TODO: cacheType, cacheSize validation - case FrameTypeInt: + case FieldTypeInt: if o.Min > o.Max { return ErrInvalidBSIGroupRange } - case FrameTypeTime: + case FieldTypeTime: if o.TimeQuantum == "" || !o.TimeQuantum.Valid() { return ErrInvalidTimeQuantum } @@ -1038,11 +1038,11 @@ func (o *FrameOptions) Validate() error { } // Encode converts o into its internal representation. -func (o *FrameOptions) Encode() *internal.FrameMeta { - return encodeFrameOptions(o) +func (o *FieldOptions) Encode() *internal.FrameMeta { + return encodeFieldOptions(o) } -func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { +func encodeFieldOptions(o *FieldOptions) *internal.FrameMeta { if o == nil { return nil } @@ -1056,11 +1056,11 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { } } -func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { +func decodeFieldOptions(options *internal.FrameMeta) *FieldOptions { if options == nil { return nil } - return &FrameOptions{ + return &FieldOptions{ Type: options.Type, CacheType: options.CacheType, CacheSize: options.CacheSize, diff --git a/frame_test.go b/frame_test.go index 3964af785..99dea0938 100644 --- a/frame_test.go +++ b/frame_test.go @@ -50,10 +50,10 @@ func TestFrame_CreateViewIfNotExists(t *testing.T) { // Ensure frame can set its time quantum. func TestFrame_SetTimeQuantum(t *testing.T) { - fo := pilosa.FrameOptions{ + fo := pilosa.FieldOptions{ Type: "time", } - f := test.MustOpenFrame(pilosa.OptFrameFrameOptions(fo)) + f := test.MustOpenFrame(pilosa.OptFieldFieldOptions(fo)) defer f.Close() // Set & retrieve time quantum. @@ -77,8 +77,8 @@ func TestFrame_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + f, err := idx.CreateFrame("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 0, Max: 30, }) @@ -114,8 +114,8 @@ func TestFrame_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + f, err := idx.CreateFrame("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 0, Max: 30, }) @@ -151,8 +151,8 @@ func TestFrame_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeSet, + f, err := idx.CreateFrame("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeSet, }) if err != nil { t.Fatal(err) @@ -168,8 +168,8 @@ func TestFrame_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + f, err := idx.CreateFrame("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 20, Max: 30, }) @@ -187,8 +187,8 @@ func TestFrame_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + f, err := idx.CreateFrame("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 20, Max: 30, }) @@ -208,7 +208,7 @@ func TestFrame_NameRestriction(t *testing.T) { if err != nil { panic(err) } - frame, err := pilosa.NewFrame(path, "i", ".meta") + frame, err := pilosa.NewField(path, "i", ".meta") if frame != nil { t.Fatalf("unexpected frame name %s", err) } @@ -240,13 +240,13 @@ func TestFrame_NameValidation(t *testing.T) { panic(err) } for _, name := range validFrameNames { - _, err := pilosa.NewFrame(path, "i", name) + _, err := pilosa.NewField(path, "i", name) if err != nil { t.Fatalf("unexpected frame name: %s %s", name, err) } } for _, name := range invalidFrameNames { - _, err := pilosa.NewFrame(path, "i", name) + _, err := pilosa.NewField(path, "i", name) if err == nil { t.Fatalf("expected error on frame name: %s", name) } diff --git a/handler.go b/handler.go index d4ed5ab37..4cc64fbb2 100644 --- a/handler.go +++ b/handler.go @@ -546,7 +546,7 @@ func (p *postFrameRequest) UnmarshalJSON(b []byte) error { return errors.Wrap(err, "unmarshaling unexpected keys") } - validFrameOptions := getValidOptions(FrameOptions{}) + validFrameOptions := getValidOptions(FieldOptions{}) err := validateOptions(m, validFrameOptions) if err != nil { return err @@ -575,7 +575,7 @@ func getValidOptions(option interface{}) []string { } type postFrameRequest struct { - Options FrameOptions `json:"options"` + Options FieldOptions `json:"options"` } type postFrameResponse struct{} diff --git a/handler_internal_test.go b/handler_internal_test.go index ceaeb707c..9adab16ac 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -62,12 +62,12 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) { expected postFrameRequest err string }{ - {json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}}, + {json: `{"options": {}}`, expected: postFrameRequest{Options: FieldOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, {json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"}, - {json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{CacheType: "type"}}}, + {json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FieldOptions{CacheType: "type"}}}, {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, } for _, test := range tests { diff --git a/handler_test.go b/handler_test.go index 30bba9254..c378a30cf 100644 --- a/handler_test.go +++ b/handler_test.go @@ -83,17 +83,17 @@ func TestHandler_Schema(t *testing.T) { i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { + if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -120,17 +120,17 @@ func TestHandler_Status(t *testing.T) { i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { + if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -699,7 +699,7 @@ func TestHandler_DeleteFrame(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { + if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -777,7 +777,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { // Set attributes on the index. idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("meta", pilosa.FrameOptions{}) + f, err := idx.CreateFrameIfNotExists("meta", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } diff --git a/holder.go b/holder.go index 6d13159b9..3a3bcca0c 100644 --- a/holder.go +++ b/holder.go @@ -215,14 +215,14 @@ func (h *Holder) Schema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{Name: index.Name()} for _, frame := range index.Frames() { - fi := &FrameInfo{Name: frame.Name(), Options: frame.Options()} + fi := &FieldInfo{Name: frame.Name(), Options: frame.Options()} for _, view := range frame.Views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) } sort.Sort(viewInfoSlice(fi.Views)) di.Frames = append(di.Frames, fi) } - sort.Sort(frameInfoSlice(di.Frames)) + sort.Sort(fieldInfoSlice(di.Frames)) a = append(a, di) } sort.Sort(indexInfoSlice(a)) @@ -240,7 +240,7 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { } // Create frames that don't exist. for _, f := range index.Frames { - opt := decodeFrameOptions(f.Meta) + opt := decodeFieldOptions(f.Meta) frame, err := idx.CreateFrameIfNotExists(f.Name, *opt) if err != nil { return errors.Wrap(err, "creating frame") @@ -391,7 +391,7 @@ func (h *Holder) DeleteIndex(name string) error { } // Frame returns the frame for an index and name. -func (h *Holder) Frame(index, name string) *Frame { +func (h *Holder) Frame(index, name string) *Field { idx := h.Index(index) if idx == nil { return nil diff --git a/holder_test.go b/holder_test.go index c7877e3ba..0081b105c 100644 --- a/holder_test.go +++ b/holder_test.go @@ -100,7 +100,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if _, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -119,7 +119,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if _, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -137,7 +137,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if _, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -159,7 +159,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if frame, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) @@ -183,7 +183,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if frame, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) @@ -208,7 +208,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if frame, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) @@ -231,7 +231,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if frame, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) @@ -257,7 +257,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if frame, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) diff --git a/index.go b/index.go index 7829903aa..8421f8273 100644 --- a/index.go +++ b/index.go @@ -35,7 +35,7 @@ type Index struct { name string // Frames by name. - frames map[string]*Frame + frames map[string]*Field // Max Slice on any node in the cluster, according to this node. remoteMaxSlice uint64 @@ -61,7 +61,7 @@ func NewIndex(path, name string) (*Index, error) { return &Index{ path: path, name: name, - frames: make(map[string]*Frame), + frames: make(map[string]*Field), remoteMaxSlice: 0, @@ -203,7 +203,7 @@ func (i *Index) Close() error { return errors.Wrap(err, "closing frame") } } - i.frames = make(map[string]*Frame) + i.frames = make(map[string]*Field) return nil } @@ -238,24 +238,24 @@ func (i *Index) SetRemoteMaxSlice(newmax uint64) { func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } // Frame returns a frame in the index by name. -func (i *Index) Frame(name string) *Frame { +func (i *Index) Frame(name string) *Field { i.mu.RLock() defer i.mu.RUnlock() return i.frame(name) } -func (i *Index) frame(name string) *Frame { return i.frames[name] } +func (i *Index) frame(name string) *Field { return i.frames[name] } // Frames returns a list of all frames in the index. -func (i *Index) Frames() []*Frame { +func (i *Index) Frames() []*Field { i.mu.RLock() defer i.mu.RUnlock() - a := make([]*Frame, 0, len(i.frames)) + a := make([]*Field, 0, len(i.frames)) for _, f := range i.frames { a = append(a, f) } - sort.Sort(frameSlice(a)) + sort.Sort(fieldSlice(a)) return a } @@ -268,7 +268,7 @@ func (i *Index) RecalculateCaches() { } // CreateFrame creates a frame. -func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) { +func (i *Index) CreateFrame(name string, opt FieldOptions) (*Field, error) { i.mu.Lock() defer i.mu.Unlock() @@ -280,7 +280,7 @@ func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) { } // CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. -func (i *Index) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) { +func (i *Index) CreateFrameIfNotExists(name string, opt FieldOptions) (*Field, error) { i.mu.Lock() defer i.mu.Unlock() @@ -292,7 +292,7 @@ func (i *Index) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, e return i.createFrame(name, opt) } -func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { +func (i *Index) createFrame(name string, opt FieldOptions) (*Field, error) { if name == "" { return nil, errors.New("frame name required") } else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) { @@ -332,8 +332,8 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return f, nil } -func (i *Index) newFrame(path, name string) (*Frame, error) { - f, err := NewFrame(path, i.name, name) +func (i *Index) newFrame(path, name string) (*Field, error) { + f, err := NewField(path, i.name, name) if err != nil { return nil, err } @@ -380,7 +380,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { Name string `json:"name"` - Frames []*FrameInfo `json:"frames"` + Frames []*FieldInfo `json:"frames"` } type indexInfoSlice []*IndexInfo @@ -402,7 +402,7 @@ func EncodeIndexes(a []*Index) []*internal.Index { func encodeIndex(d *Index) *internal.Index { return &internal.Index{ Name: d.name, - Frames: encodeFrames(d.Frames()), + Frames: encodeFields(d.Frames()), } } diff --git a/index_test.go b/index_test.go index 86d9043ce..be6288ad8 100644 --- a/index_test.go +++ b/index_test.go @@ -29,7 +29,7 @@ func TestIndex_CreateFrameIfNotExists(t *testing.T) { defer index.Close() // Create frame. - f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) + f, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } else if f == nil { @@ -37,14 +37,14 @@ func TestIndex_CreateFrameIfNotExists(t *testing.T) { } // Retrieve existing frame. - other, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) + other, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) - } else if f.Frame != other.Frame { + } else if f.Field != other.Field { t.Fatal("frame mismatch") } - if f.Frame != index.Frame("f") { + if f.Field != index.Frame("f") { t.Fatal("frame mismatch") } } @@ -57,8 +57,8 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() // Create frame with explicit quantum. - f, err := index.CreateFrame("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeTime, + f, err := index.CreateFrame("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeTime, TimeQuantum: pilosa.TimeQuantum("YMDH"), }) if err != nil { @@ -76,20 +76,20 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() // Create frame with schema and verify it exists. - if f, err := index.CreateFrame("f", pilosa.FrameOptions{ - Type: pilosa.FrameTypeInt, + if f, err := index.CreateFrame("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, Min: 10, Max: 20, }); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(f.Type(), pilosa.FrameTypeInt) { + } else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { t.Fatalf("unexpected type: %#v", f.Type()) } // Reopen the index & verify the fields are loaded. if err := index.Reopen(); err != nil { t.Fatal(err) - } else if f := index.Frame("f"); !reflect.DeepEqual(f.Type(), pilosa.FrameTypeInt) { + } else if f := index.Frame("f"); !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { t.Fatalf("unexpected type after reopen: %#v", f.Type()) } }) @@ -180,7 +180,7 @@ func TestIndex_DeleteFrame(t *testing.T) { defer index.Close() // Create frame. - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + if _, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } diff --git a/server.go b/server.go index 02e56e9b0..d0585c992 100644 --- a/server.go +++ b/server.go @@ -460,7 +460,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - opt := decodeFrameOptions(obj.Meta) + opt := decodeFieldOptions(obj.Meta) _, err := idx.CreateFrame(obj.Frame, *opt) if err != nil { return err diff --git a/server/cluster_test.go b/server/cluster_test.go index e6777d90c..1b4e0b807 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -54,7 +54,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Create indexes and frames on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -209,7 +209,7 @@ func TestClusterResize_AddNode(t *testing.T) { // Create indexes and frames on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -253,7 +253,7 @@ func TestClusterResize_AddNode(t *testing.T) { // Create indexes and frames on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -305,7 +305,7 @@ func TestClusterResize_AddNode(t *testing.T) { // Create indexes and frames on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -458,7 +458,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // Create indexes and frames on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } diff --git a/server/server_test.go b/server/server_test.go index edac4b6fd..920fde809 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -54,7 +54,7 @@ func TestMain_Set_Quick(t *testing.T) { if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && !strings.Contains(err.Error(), "index already exists") { t.Fatal(err) } - if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && !strings.Contains(err.Error(), "frame already exists") { + if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FieldOptions{}); err != nil && !strings.Contains(err.Error(), "frame already exists") { t.Fatal(err) } if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { @@ -123,11 +123,11 @@ func TestMain_SetRowAttrs(t *testing.T) { client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "z", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "z", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "neg", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "neg", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -200,7 +200,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -286,7 +286,7 @@ func TestMain_RecalculateHashes(t *testing.T) { if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal("create index:", err) } - if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{CacheType: "ranked"}); err != nil { + if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{CacheType: "ranked"}); err != nil { t.Fatal("create frame:", err) } diff --git a/server_test.go b/server_test.go index 9e10374d7..9b5409330 100644 --- a/server_test.go +++ b/server_test.go @@ -19,7 +19,7 @@ func TestMonitorAntiEntropy(t *testing.T) { if err != nil { t.Fatalf("creating index: %v", err) } - err = client.CreateFrame(context.Background(), "balh", "fralh", pilosa.FrameOptions{}) + err = client.CreateFrame(context.Background(), "balh", "fralh", pilosa.FieldOptions{}) if err != nil { t.Fatalf("creating frame: %v", err) } diff --git a/stats_test.go b/stats_test.go index 611b7b2b6..47521fbf8 100644 --- a/stats_test.go +++ b/stats_test.go @@ -298,7 +298,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { called := false // Create index. indx, _ := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := indx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}); err != nil { + if _, err := indx.CreateFrameIfNotExists("test", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } s.Handler.API.Holder.Stats = &MockStats{ diff --git a/test/cluster.go b/test/cluster.go index bf6858e79..4c0bd45a0 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -114,7 +114,7 @@ func (t *TestCluster) CreateIndex(name string) error { return nil } -func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FrameOptions) error { +func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FieldOptions) error { for _, c := range t.Clusters { idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{}) if err != nil { diff --git a/test/frame.go b/test/frame.go index 09503a3cf..929070368 100644 --- a/test/frame.go +++ b/test/frame.go @@ -25,24 +25,24 @@ import ( // Frame represents a test wrapper for pilosa.Frame. type Frame struct { - *pilosa.Frame + *pilosa.Field } // NewFrame returns a new instance of Frame d/0. -func NewFrame(opt ...pilosa.FrameOption) *Frame { +func NewFrame(opt ...pilosa.FieldOption) *Frame { path, err := ioutil.TempDir("", "pilosa-frame-") if err != nil { panic(err) } - frame, err := pilosa.NewFrame(path, "i", "f", opt...) + frame, err := pilosa.NewField(path, "i", "f", opt...) if err != nil { panic(err) } - return &Frame{Frame: frame} + return &Frame{Field: frame} } // MustOpenFrame returns a new, opened frame at a temporary path. Panic on error. -func MustOpenFrame(opt ...pilosa.FrameOption) *Frame { +func MustOpenFrame(opt ...pilosa.FieldOption) *Frame { f := NewFrame(opt...) if err := f.Open(); err != nil { panic(err) @@ -53,18 +53,18 @@ func MustOpenFrame(opt ...pilosa.FrameOption) *Frame { // Close closes the frame and removes the underlying data. func (f *Frame) Close() error { defer os.RemoveAll(f.Path()) - return f.Frame.Close() + return f.Field.Close() } // Reopen closes the index and reopens it. func (f *Frame) Reopen() error { var err error - if err := f.Frame.Close(); err != nil { + if err := f.Field.Close(); err != nil { return err } path, index, name := f.Path(), f.Index(), f.Name() - f.Frame, err = pilosa.NewFrame(path, index, name) + f.Field, err = pilosa.NewField(path, index, name) if err != nil { return err } diff --git a/test/holder.go b/test/holder.go index 97b399bea..1344bd4c9 100644 --- a/test/holder.go +++ b/test/holder.go @@ -82,7 +82,7 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption // MustCreateFrameIfNotExists returns a given frame. Panic on error. func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame { - f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) + f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FieldOptions{}) if err != nil { panic(err) } @@ -92,7 +92,7 @@ func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame { // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) + f, err := idx.CreateFrameIfNotExists(frame, pilosa.FieldOptions{}) if err != nil { panic(err) } @@ -110,7 +110,7 @@ func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice // MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked}) + f, err := idx.CreateFrameIfNotExists(frame, pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) if err != nil { panic(err) } diff --git a/test/index.go b/test/index.go index 5354c85f9..5689afda3 100644 --- a/test/index.go +++ b/test/index.go @@ -74,19 +74,19 @@ func (i *Index) Reopen() error { } // CreateFrame creates a frame with the given options. -func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) { +func (i *Index) CreateFrame(name string, opt pilosa.FieldOptions) (*Frame, error) { f, err := i.Index.CreateFrame(name, opt) if err != nil { return nil, err } - return &Frame{Frame: f}, nil + return &Frame{Field: f}, nil } // CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. -func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) { +func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FieldOptions) (*Frame, error) { f, err := i.Index.CreateFrameIfNotExists(name, opt) if err != nil { return nil, err } - return &Frame{Frame: f}, nil + return &Frame{Field: f}, nil } diff --git a/utils_test.go b/utils_test.go index eb4170a19..7773db06c 100644 --- a/utils_test.go +++ b/utils_test.go @@ -104,7 +104,7 @@ func (t *ClusterCluster) CreateIndex(name string) error { return nil } -func (t *ClusterCluster) CreateFrame(index, frame string, opt FrameOptions) error { +func (t *ClusterCluster) CreateFrame(index, frame string, opt FieldOptions) error { for _, c := range t.Clusters { idx, err := c.Holder.CreateIndexIfNotExists(index, IndexOptions{}) if err != nil { From 3531c128c4fddaee2c16d5373fbc8925fc8e05a5 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 22:33:48 -0500 Subject: [PATCH 36/78] GoRename Frame to Field in index.go --- api.go | 18 ++++---- client.go | 14 +++---- cluster.go | 4 +- cluster_internal_test.go | 2 +- ctl/export.go | 2 +- ctl/export_test.go | 4 +- ctl/import.go | 4 +- ctl/import_test.go | 4 +- diagnostics.go | 2 +- executor.go | 26 ++++++------ executor_test.go | 28 ++++++------- fragment_test.go | 2 +- handler.go | 6 +-- handler_test.go | 2 +- holder.go | 20 ++++----- holder_test.go | 16 ++++---- index.go | 88 ++++++++++++++++++++-------------------- index_test.go | 10 ++--- pilosa.go | 8 ++-- server.go | 4 +- server/cluster_test.go | 4 +- stats_test.go | 2 +- test/cluster.go | 2 +- test/index.go | 4 +- utils_test.go | 2 +- 25 files changed, 139 insertions(+), 139 deletions(-) diff --git a/api.go b/api.go index fec6b3129..d47ba1ff4 100644 --- a/api.go +++ b/api.go @@ -230,7 +230,7 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str } // Create frame. - frame, err := index.CreateFrame(frameName, options) + frame, err := index.CreateField(frameName, options) if err != nil { return nil, errors.Wrap(err, "creating frame") } @@ -265,7 +265,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str } // Delete frame from the index. - if err := index.DeleteFrame(frameName); err != nil { + if err := index.DeleteField(frameName); err != nil { return errors.Wrap(err, "deleting frame") } @@ -357,7 +357,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa // Retrieve frame. f := api.Holder.Frame(indexName, frameName) if f == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Retrieve view. @@ -497,7 +497,7 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( // Retrieve views. f := api.Holder.Frame(indexName, frameName) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } // Fetch views. @@ -514,7 +514,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // Retrieve frame. f := api.Holder.Frame(indexName, frameName) if f == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Delete the view. @@ -582,7 +582,7 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s // Retrieve index from holder. f := api.Holder.Frame(indexName, frameName) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } // Retrieve local blocks. @@ -695,10 +695,10 @@ func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*I } // Retrieve frame. - frame := index.Frame(frameName) + frame := index.Field(frameName) if frame == nil { - api.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error()) - return nil, nil, ErrFrameNotFound + api.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFieldNotFound.Error()) + return nil, nil, ErrFieldNotFound } return index, frame, nil } diff --git a/client.go b/client.go index 4c8ba2157..d186746c7 100644 --- a/client.go +++ b/client.go @@ -273,7 +273,7 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl if index == "" { return ErrIndexRequired } else if frame == "" { - return ErrFrameRequired + return ErrFieldRequired } buf, err := marshalImportPayload(index, frame, slice, bits) @@ -302,7 +302,7 @@ func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, c if index == "" { return ErrIndexRequired } else if frame == "" { - return ErrFrameRequired + return ErrFieldRequired } buf, err := marshalImportPayloadK(index, frame, columns) @@ -332,7 +332,7 @@ func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, optio func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, frameName string, options FieldOptions) error { err := c.CreateFrame(ctx, indexName, frameName, options) - if err == nil || err == ErrFrameExists { + if err == nil || err == ErrFieldExists { return nil } return err @@ -424,7 +424,7 @@ func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame strin if index == "" { return ErrIndexRequired } else if frame == "" { - return ErrFrameRequired + return ErrFieldRequired } buf, err := marshalImportValuePayload(index, frame, slice, vals) @@ -511,7 +511,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame string, if index == "" { return ErrIndexRequired } else if frame == "" { - return ErrFrameRequired + return ErrFieldRequired } // Retrieve a list of nodes that own the slice. @@ -658,7 +658,7 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin case http.StatusOK: return nil // ok case http.StatusConflict: - return ErrFrameExists + return ErrFieldExists default: return errors.New(string(body)) } @@ -822,7 +822,7 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin switch resp.StatusCode { case http.StatusOK: // ok case http.StatusNotFound: - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound default: return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) } diff --git a/cluster.go b/cluster.go index 63f4f62b8..deabf2d06 100644 --- a/cluster.go +++ b/cluster.go @@ -627,7 +627,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost { // frameViews is a map of frame to slice of views. frameViews := make(viewsByFrame) - for _, frame := range idx.Frames() { + for _, frame := range idx.Fields() { for _, view := range frame.Views() { frameViews.addView(frame.Name(), view.Name()) @@ -1242,7 +1242,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Retrieve frame. f := c.Holder.Frame(src.Index, src.Frame) if f == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Create view. diff --git a/cluster_internal_test.go b/cluster_internal_test.go index acccd37d3..9453fc16b 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -145,7 +145,7 @@ func TestFragSources(t *testing.T) { c5.addNodeBasicSorted(node3) idx := newIndexWithTempPath("i") - frame, err := idx.CreateFrameIfNotExists("f", FieldOptions{}) + frame, err := idx.CreateFieldIfNotExists("f", FieldOptions{}) if err != nil { t.Fatal(err) } diff --git a/ctl/export.go b/ctl/export.go index 5ee8d2120..988403272 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -58,7 +58,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { if cmd.Index == "" { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { - return pilosa.ErrFrameRequired + return pilosa.ErrFieldRequired } // Use output file, if specified. diff --git a/ctl/export_test.go b/ctl/export_test.go index 469ca7d75..6948b810f 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -38,8 +38,8 @@ func TestExportCommand_Validation(t *testing.T) { cm.Index = "i" err = cm.Run(context.Background()) - if err != pilosa.ErrFrameRequired { - t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err) + if err != pilosa.ErrFieldRequired { + t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err) } } diff --git a/ctl/import.go b/ctl/import.go index 3a7835cb2..68229ed72 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -84,7 +84,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { if cmd.Index == "" { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { - return pilosa.ErrFrameRequired + return pilosa.ErrFieldRequired } else if len(cmd.Paths) == 0 { return errors.New("path required") } @@ -110,7 +110,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } for _, index := range schema { if index.Name == cmd.Index { - for _, frame := range index.Frames { + for _, frame := range index.Fields { if frame.Name == cmd.Frame { frameType = frame.Options.Type } diff --git a/ctl/import_test.go b/ctl/import_test.go index 7c64944f6..9b6dd3c6e 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -39,8 +39,8 @@ func TestImportCommand_Validation(t *testing.T) { cm.Index = "i" err = cm.Run(context.Background()) - if err != pilosa.ErrFrameRequired { - t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err) + if err != pilosa.ErrFieldRequired { + t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err) } cm.Frame = "f" diff --git a/diagnostics.go b/diagnostics.go index 10ecfa22d..f97f5f6aa 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -225,7 +225,7 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { for _, index := range d.server.Holder.Indexes() { numSlices += index.MaxSlice() + 1 numIndexes += 1 - for _, frame := range index.Frames() { + for _, frame := range index.Fields() { numFrames += 1 if frame.Type() == FieldTypeInt { bsiFieldCount += 1 diff --git a/executor.go b/executor.go index fd98f7189..b3e4f5aad 100644 --- a/executor.go +++ b/executor.go @@ -313,7 +313,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C return nil, err } else { frame, _ := c.Args["frame"].(string) - if fr := idx.Frame(frame); fr != nil { + if fr := idx.Field(frame); fr != nil { rowID, _, err := c.UintArg(rowLabel) if err != nil { return nil, errors.Wrap(err, "getting row") @@ -634,7 +634,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. } f := e.Holder.Frame(index, frame) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } rowID, rowOK, rowErr := c.UintArg(rowLabel) @@ -694,9 +694,9 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Retrieve base frame. - f := idx.Frame(frame) + f := idx.Field(frame) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } // Read row & column id. @@ -769,7 +769,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, f := e.Holder.Frame(index, frameName) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } // EQ null (not implemented: flip frag.NotNull with max ColumnID) @@ -959,9 +959,9 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal if idx == nil { return false, ErrIndexNotFound } - f := idx.Frame(frame) + f := idx.Field(frame) if f == nil { - return false, ErrFrameNotFound + return false, ErrFieldNotFound } // Read fields using labels. @@ -1024,9 +1024,9 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, if idx == nil { return false, ErrIndexNotFound } - f := idx.Frame(frame) + f := idx.Field(frame) if f == nil { - return false, ErrFrameNotFound + return false, ErrFieldNotFound } // Read fields using labels. @@ -1110,7 +1110,7 @@ func (e *Executor) executeSetValue(ctx context.Context, index string, c *pql.Cal // Retrieve frame. frame := e.Holder.Frame(index, name) if frame == nil { - return ErrFrameNotFound + return ErrFieldNotFound } switch value := value.(type) { @@ -1159,7 +1159,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. // Retrieve frame. frame := e.Holder.Frame(index, frameName) if frame == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Parse labels. @@ -1219,7 +1219,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // Retrieve frame. f := e.Holder.Frame(index, frame) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } rowID, ok, err := c.UintArg(rowLabel) @@ -1257,7 +1257,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // Retrieve frame. frame := e.Holder.Frame(index, name) if frame == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } // Set attributes. diff --git a/executor_test.go b/executor_test.go index 0d70ae37f..b8f7aa317 100644 --- a/executor_test.go +++ b/executor_test.go @@ -388,9 +388,9 @@ func TestExecutor_Execute_TopN(t *testing.T) { // Set columns for rows 0, 10, & 20 across two slices. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("f", pilosa.FieldOptions{}); err != nil { + } else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("other", pilosa.FieldOptions{}); err != nil { + } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` SetBit(frame=f, row=0, col=0) @@ -572,11 +572,11 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("x", pilosa.FieldOptions{}); err != nil { + if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FieldOptions{ + if _, err := idx.CreateField("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: -10, Max: 100, @@ -667,11 +667,11 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("x", pilosa.FieldOptions{}); err != nil { + if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("foo", pilosa.FieldOptions{ + if _, err := idx.CreateField("foo", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 10, Max: 100, @@ -679,7 +679,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("bar", pilosa.FieldOptions{ + if _, err := idx.CreateField("bar", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 0, Max: 100000, @@ -687,7 +687,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("other", pilosa.FieldOptions{ + if _, err := idx.CreateField("other", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 0, Max: 1000, @@ -780,11 +780,11 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FieldOptions{}); err != nil { + if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("foo", pilosa.FieldOptions{ + if _, err := idx.CreateField("foo", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 10, Max: 100, @@ -792,7 +792,7 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("bar", pilosa.FieldOptions{ + if _, err := idx.CreateField("bar", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 0, Max: 100000, @@ -800,7 +800,7 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("other", pilosa.FieldOptions{ + if _, err := idx.CreateField("other", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 0, Max: 1000, @@ -808,7 +808,7 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateFrame("edge", pilosa.FieldOptions{ + if _, err := idx.CreateField("edge", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: -100, Max: 100, @@ -945,7 +945,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("ErrFrameNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(bad_frame >= 20)`), nil, nil); err != pilosa.ErrFrameNotFound { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(bad_frame >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { t.Fatal(err) } }) diff --git a/fragment_test.go b/fragment_test.go index 2d618aa81..71dff2b71 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -961,7 +961,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { } // Re-fetch fragment. - f = index.Frame("f").View(pilosa.ViewStandard).Fragment(0) + f = index.Field("f").View(pilosa.ViewStandard).Fragment(0) // Re-verify correct cache type and size. if cache, ok := f.Cache().(*pilosa.RankCache); !ok { diff --git a/handler.go b/handler.go index 4cc64fbb2..a57a40204 100644 --- a/handler.go +++ b/handler.go @@ -522,7 +522,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { switch errors.Cause(err) { case ErrIndexNotFound: http.Error(w, err.Error(), http.StatusNotFound) - case ErrFrameExists: + case ErrFieldExists: http.Error(w, err.Error(), http.StatusConflict) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -749,7 +749,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { switch errors.Cause(err) { case ErrIndexNotFound: fallthrough - case ErrFrameNotFound: + case ErrFieldNotFound: http.Error(w, err.Error(), http.StatusNotFound) case ErrClusterDoesNotOwnSlice: http.Error(w, err.Error(), http.StatusPreconditionFailed) @@ -802,7 +802,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) switch errors.Cause(err) { case ErrIndexNotFound: fallthrough - case ErrFrameNotFound: + case ErrFieldNotFound: http.Error(w, err.Error(), http.StatusNotFound) case ErrClusterDoesNotOwnSlice: http.Error(w, err.Error(), http.StatusPreconditionFailed) diff --git a/handler_test.go b/handler_test.go index c378a30cf..74b1702fa 100644 --- a/handler_test.go +++ b/handler_test.go @@ -712,7 +712,7 @@ func TestHandler_DeleteFrame(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) - } else if f := hldr.Index("i0").Frame("f1"); f != nil { + } else if f := hldr.Index("i0").Field("f1"); f != nil { t.Fatal("expected nil frame") } } diff --git a/holder.go b/holder.go index 3a3bcca0c..f0c6d027c 100644 --- a/holder.go +++ b/holder.go @@ -214,15 +214,15 @@ func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{Name: index.Name()} - for _, frame := range index.Frames() { + for _, frame := range index.Fields() { fi := &FieldInfo{Name: frame.Name(), Options: frame.Options()} for _, view := range frame.Views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) } sort.Sort(viewInfoSlice(fi.Views)) - di.Frames = append(di.Frames, fi) + di.Fields = append(di.Fields, fi) } - sort.Sort(fieldInfoSlice(di.Frames)) + sort.Sort(fieldInfoSlice(di.Fields)) a = append(a, di) } sort.Sort(indexInfoSlice(a)) @@ -241,7 +241,7 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { // Create frames that don't exist. for _, f := range index.Frames { opt := decodeFieldOptions(f.Meta) - frame, err := idx.CreateFrameIfNotExists(f.Name, *opt) + frame, err := idx.CreateFieldIfNotExists(f.Name, *opt) if err != nil { return errors.Wrap(err, "creating frame") } @@ -396,7 +396,7 @@ func (h *Holder) Frame(index, name string) *Field { if idx == nil { return nil } - return idx.Frame(name) + return idx.Field(name) } // View returns the view for an index, frame, and name. @@ -435,7 +435,7 @@ func (h *Holder) monitorCacheFlush() { func (h *Holder) flushCaches() { for _, index := range h.Indexes() { - for _, frame := range index.Frames() { + for _, frame := range index.Fields() { for _, view := range frame.Views() { for _, fragment := range view.Fragments() { select { @@ -600,7 +600,7 @@ func (s *HolderSyncer) SyncHolder() error { } tf := time.Now() - for _, fi := range di.Frames { + for _, fi := range di.Fields { // Verify syncer has not closed. if s.IsClosing() { return nil @@ -713,7 +713,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. m, err := client.RowAttrDiff(context.Background(), index, name, blks) - if err == ErrFrameNotFound { + if err == ErrFieldNotFound { continue // frame not created remotely yet, skip } else if err != nil { return errors.Wrap(err, "getting differing blocks") @@ -742,7 +742,7 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err // Retrieve local frame. f := s.Holder.Frame(index, frame) if f == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Ensure view exists locally. @@ -806,7 +806,7 @@ func (c *HolderCleaner) CleanHolder() error { containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node) // Get the fragments registered in memory. - for _, frame := range index.Frames() { + for _, frame := range index.Fields() { for _, view := range frame.Views() { for _, fragment := range view.Fragments() { fragSlice := fragment.Slice() diff --git a/holder_test.go b/holder_test.go index 0081b105c..664bf16a0 100644 --- a/holder_test.go +++ b/holder_test.go @@ -100,7 +100,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -119,7 +119,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -137,7 +137,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -159,7 +159,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { + } else if frame, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) @@ -183,7 +183,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { + } else if frame, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) @@ -208,7 +208,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { + } else if frame, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) @@ -231,7 +231,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { + } else if frame, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) @@ -257,7 +257,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FieldOptions{}); err != nil { + } else if frame, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) diff --git a/index.go b/index.go index 8421f8273..727989914 100644 --- a/index.go +++ b/index.go @@ -34,8 +34,8 @@ type Index struct { path string name string - // Frames by name. - frames map[string]*Field + // Fields by name. + fields map[string]*Field // Max Slice on any node in the cluster, according to this node. remoteMaxSlice uint64 @@ -61,7 +61,7 @@ func NewIndex(path, name string) (*Index, error) { return &Index{ path: path, name: name, - frames: make(map[string]*Field), + fields: make(map[string]*Field), remoteMaxSlice: 0, @@ -106,7 +106,7 @@ func (i *Index) Open() error { return errors.Wrap(err, "loading meta file") } - if err := i.openFrames(); err != nil { + if err := i.openFields(); err != nil { return errors.Wrap(err, "opening frames") } @@ -117,8 +117,8 @@ func (i *Index) Open() error { return nil } -// openFrames opens and initializes the frames inside the index. -func (i *Index) openFrames() error { +// openFields opens and initializes the frames inside the index. +func (i *Index) openFields() error { f, err := os.Open(i.path) if err != nil { return errors.Wrap(err, "opening directory") @@ -135,14 +135,14 @@ func (i *Index) openFrames() error { continue } - fr, err := i.newFrame(i.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + fld, err := i.newField(i.FieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err != nil { return ErrName } - if err := fr.Open(); err != nil { - return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err) + if err := fld.Open(); err != nil { + return fmt.Errorf("open frame: name=%s, err=%s", fld.Name(), err) } - i.frames[fr.Name()] = fr + i.fields[fld.Name()] = fld } return nil } @@ -198,12 +198,12 @@ func (i *Index) Close() error { i.columnAttrStore.Close() // Close all frames. - for _, f := range i.frames { + for _, f := range i.fields { if err := f.Close(); err != nil { return errors.Wrap(err, "closing frame") } } - i.frames = make(map[string]*Field) + i.fields = make(map[string]*Field) return nil } @@ -217,7 +217,7 @@ func (i *Index) MaxSlice() uint64 { defer i.mu.RUnlock() max := i.remoteMaxSlice - for _, f := range i.frames { + for _, f := range i.fields { if slice := f.MaxSlice(); slice > max { max = slice } @@ -234,25 +234,25 @@ func (i *Index) SetRemoteMaxSlice(newmax uint64) { i.remoteMaxSlice = newmax } -// FramePath returns the path to a frame in the index. -func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } +// FieldPath returns the path to a field in the index. +func (i *Index) FieldPath(name string) string { return filepath.Join(i.path, name) } -// Frame returns a frame in the index by name. -func (i *Index) Frame(name string) *Field { +// Field returns a frame in the index by name. +func (i *Index) Field(name string) *Field { i.mu.RLock() defer i.mu.RUnlock() - return i.frame(name) + return i.field(name) } -func (i *Index) frame(name string) *Field { return i.frames[name] } +func (i *Index) field(name string) *Field { return i.fields[name] } -// Frames returns a list of all frames in the index. -func (i *Index) Frames() []*Field { +// Fields returns a list of all fields in the index. +func (i *Index) Fields() []*Field { i.mu.RLock() defer i.mu.RUnlock() - a := make([]*Field, 0, len(i.frames)) - for _, f := range i.frames { + a := make([]*Field, 0, len(i.fields)) + for _, f := range i.fields { a = append(a, f) } sort.Sort(fieldSlice(a)) @@ -262,37 +262,37 @@ func (i *Index) Frames() []*Field { // RecalculateCaches recalculates caches on every frame in the index. func (i *Index) RecalculateCaches() { - for _, frame := range i.Frames() { + for _, frame := range i.Fields() { frame.RecalculateCaches() } } -// CreateFrame creates a frame. -func (i *Index) CreateFrame(name string, opt FieldOptions) (*Field, error) { +// CreateField creates a field. +func (i *Index) CreateField(name string, opt FieldOptions) (*Field, error) { i.mu.Lock() defer i.mu.Unlock() // Ensure frame doesn't already exist. - if i.frames[name] != nil { - return nil, ErrFrameExists + if i.fields[name] != nil { + return nil, ErrFieldExists } - return i.createFrame(name, opt) + return i.createField(name, opt) } -// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. -func (i *Index) CreateFrameIfNotExists(name string, opt FieldOptions) (*Field, error) { +// CreateFieldIfNotExists creates a field with the given options if it doesn't exist. +func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, error) { i.mu.Lock() defer i.mu.Unlock() // Find frame in cache first. - if f := i.frames[name]; f != nil { + if f := i.fields[name]; f != nil { return f, nil } - return i.createFrame(name, opt) + return i.createField(name, opt) } -func (i *Index) createFrame(name string, opt FieldOptions) (*Field, error) { +func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { if name == "" { return nil, errors.New("frame name required") } else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) { @@ -305,7 +305,7 @@ func (i *Index) createFrame(name string, opt FieldOptions) (*Field, error) { } // Initialize frame. - f, err := i.newFrame(i.FramePath(name), name) + f, err := i.newField(i.FieldPath(name), name) if err != nil { return nil, errors.Wrap(err, "initializing") } @@ -327,12 +327,12 @@ func (i *Index) createFrame(name string, opt FieldOptions) (*Field, error) { } // Add to index's frame lookup. - i.frames[name] = f + i.fields[name] = f return f, nil } -func (i *Index) newFrame(path, name string) (*Field, error) { +func (i *Index) newField(path, name string) (*Field, error) { f, err := NewField(path, i.name, name) if err != nil { return nil, err @@ -344,13 +344,13 @@ func (i *Index) newFrame(path, name string) (*Field, error) { return f, nil } -// DeleteFrame removes a frame from the index. -func (i *Index) DeleteFrame(name string) error { +// DeleteField removes a field from the index. +func (i *Index) DeleteField(name string) error { i.mu.Lock() defer i.mu.Unlock() // Ignore if frame doesn't exist. - f := i.frame(name) + f := i.field(name) if f == nil { return nil } @@ -361,12 +361,12 @@ func (i *Index) DeleteFrame(name string) error { } // Delete frame directory. - if err := os.RemoveAll(i.FramePath(name)); err != nil { + if err := os.RemoveAll(i.FieldPath(name)); err != nil { return errors.Wrap(err, "removing directory") } // Remove reference. - delete(i.frames, name) + delete(i.fields, name) return nil } @@ -380,7 +380,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { Name string `json:"name"` - Frames []*FieldInfo `json:"frames"` + Fields []*FieldInfo `json:"fields"` } type indexInfoSlice []*IndexInfo @@ -402,7 +402,7 @@ func EncodeIndexes(a []*Index) []*internal.Index { func encodeIndex(d *Index) *internal.Index { return &internal.Index{ Name: d.name, - Frames: encodeFields(d.Frames()), + Frames: encodeFields(d.Fields()), } } diff --git a/index_test.go b/index_test.go index be6288ad8..ada9be7f1 100644 --- a/index_test.go +++ b/index_test.go @@ -44,7 +44,7 @@ func TestIndex_CreateFrameIfNotExists(t *testing.T) { t.Fatal("frame mismatch") } - if f.Field != index.Frame("f") { + if f.Field != index.Field("f") { t.Fatal("frame mismatch") } } @@ -89,7 +89,7 @@ func TestIndex_CreateFrame(t *testing.T) { // Reopen the index & verify the fields are loaded. if err := index.Reopen(); err != nil { t.Fatal(err) - } else if f := index.Frame("f"); !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { + } else if f := index.Field("f"); !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { t.Fatalf("unexpected type after reopen: %#v", f.Type()) } }) @@ -185,14 +185,14 @@ func TestIndex_DeleteFrame(t *testing.T) { } // Delete frame & verify it's gone. - if err := index.DeleteFrame("f"); err != nil { + if err := index.DeleteField("f"); err != nil { t.Fatal(err) - } else if index.Frame("f") != nil { + } else if index.Field("f") != nil { t.Fatal("expected nil frame") } // Delete again to make sure it doesn't error. - if err := index.DeleteFrame("f"); err != nil { + if err := index.DeleteField("f"); err != nil { t.Fatal(err) } } diff --git a/pilosa.go b/pilosa.go index 21bfdd822..2c0a8502d 100644 --- a/pilosa.go +++ b/pilosa.go @@ -31,10 +31,10 @@ var ( ErrIndexExists = errors.New("index already exists") ErrIndexNotFound = errors.New("index not found") - // ErrFrameRequired is returned when no frame is specified. - ErrFrameRequired = errors.New("frame required") - ErrFrameExists = errors.New("frame already exists") - ErrFrameNotFound = errors.New("frame not found") + // ErrFieldRequired is returned when no field is specified. + ErrFieldRequired = errors.New("field required") + ErrFieldExists = errors.New("field already exists") + ErrFieldNotFound = errors.New("field not found") ErrBSIGroupNotFound = errors.New("bsigroup not found") ErrBSIGroupExists = errors.New("bsigroup already exists") diff --git a/server.go b/server.go index d0585c992..d92c5949e 100644 --- a/server.go +++ b/server.go @@ -461,13 +461,13 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return fmt.Errorf("Local Index not found: %s", obj.Index) } opt := decodeFieldOptions(obj.Meta) - _, err := idx.CreateFrame(obj.Frame, *opt) + _, err := idx.CreateField(obj.Frame, *opt) if err != nil { return err } case *internal.DeleteFrameMessage: idx := s.Holder.Index(obj.Index) - if err := idx.DeleteFrame(obj.Frame); err != nil { + if err := idx.DeleteField(obj.Frame); err != nil { return err } case *internal.CreateViewMessage: diff --git a/server/cluster_test.go b/server/cluster_test.go index 1b4e0b807..f445ce415 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -66,7 +66,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { received0 := map[string][]string{} for _, idx := range schema0 { received0[idx.Name] = []string{} - for _, frame := range idx.Frames { + for _, frame := range idx.Fields { received0[idx.Name] = append(received0[idx.Name], frame.Name) } } @@ -82,7 +82,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { received1 := map[string][]string{} for _, idx := range schema1 { received1[idx.Name] = []string{} - for _, frame := range idx.Frames { + for _, frame := range idx.Fields { received1[idx.Name] = append(received1[idx.Name], frame.Name) } } diff --git a/stats_test.go b/stats_test.go index 47521fbf8..992055166 100644 --- a/stats_test.go +++ b/stats_test.go @@ -298,7 +298,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { called := false // Create index. indx, _ := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := indx.CreateFrameIfNotExists("test", pilosa.FieldOptions{}); err != nil { + if _, err := indx.CreateFieldIfNotExists("test", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } s.Handler.API.Holder.Stats = &MockStats{ diff --git a/test/cluster.go b/test/cluster.go index 4c0bd45a0..e293764ad 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -120,7 +120,7 @@ func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FieldOptions) if err != nil { return err } - if _, err := idx.CreateFrame(frame, opt); err != nil { + if _, err := idx.CreateField(frame, opt); err != nil { return err } } diff --git a/test/index.go b/test/index.go index 5689afda3..93fa99cad 100644 --- a/test/index.go +++ b/test/index.go @@ -75,7 +75,7 @@ func (i *Index) Reopen() error { // CreateFrame creates a frame with the given options. func (i *Index) CreateFrame(name string, opt pilosa.FieldOptions) (*Frame, error) { - f, err := i.Index.CreateFrame(name, opt) + f, err := i.Index.CreateField(name, opt) if err != nil { return nil, err } @@ -84,7 +84,7 @@ func (i *Index) CreateFrame(name string, opt pilosa.FieldOptions) (*Frame, error // CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FieldOptions) (*Frame, error) { - f, err := i.Index.CreateFrameIfNotExists(name, opt) + f, err := i.Index.CreateFieldIfNotExists(name, opt) if err != nil { return nil, err } diff --git a/utils_test.go b/utils_test.go index 7773db06c..ccb39ef54 100644 --- a/utils_test.go +++ b/utils_test.go @@ -110,7 +110,7 @@ func (t *ClusterCluster) CreateFrame(index, frame string, opt FieldOptions) erro if err != nil { return err } - if _, err := idx.CreateFrame(frame, opt); err != nil { + if _, err := idx.CreateField(frame, opt); err != nil { return err } } From 9db359d34ac36896a2cf541db3abdc628b99ffef Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 22:36:19 -0500 Subject: [PATCH 37/78] GoRename Frame to Field in holder.go --- api.go | 8 ++++---- cluster.go | 2 +- cluster_test.go | 4 ++-- executor.go | 18 +++++++++--------- executor_test.go | 8 ++++---- holder.go | 18 +++++++++--------- server.go | 4 ++-- stats_test.go | 2 +- test/cluster.go | 4 ++-- utils_test.go | 4 ++-- 10 files changed, 36 insertions(+), 36 deletions(-) diff --git a/api.go b/api.go index d47ba1ff4..9bd3abdf4 100644 --- a/api.go +++ b/api.go @@ -355,7 +355,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa } // Retrieve frame. - f := api.Holder.Frame(indexName, frameName) + f := api.Holder.Field(indexName, frameName) if f == nil { return ErrFieldNotFound } @@ -495,7 +495,7 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( } // Retrieve views. - f := api.Holder.Frame(indexName, frameName) + f := api.Holder.Field(indexName, frameName) if f == nil { return nil, ErrFieldNotFound } @@ -512,7 +512,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri } // Retrieve frame. - f := api.Holder.Frame(indexName, frameName) + f := api.Holder.Field(indexName, frameName) if f == nil { return ErrFieldNotFound } @@ -580,7 +580,7 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s } // Retrieve index from holder. - f := api.Holder.Frame(indexName, frameName) + f := api.Holder.Field(indexName, frameName) if f == nil { return nil, ErrFieldNotFound } diff --git a/cluster.go b/cluster.go index deabf2d06..a8f3573c5 100644 --- a/cluster.go +++ b/cluster.go @@ -1240,7 +1240,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err srcURI := decodeURI(src.Node.URI) // Retrieve frame. - f := c.Holder.Frame(src.Index, src.Frame) + f := c.Holder.Field(src.Index, src.Frame) if f == nil { return ErrFieldNotFound } diff --git a/cluster_test.go b/cluster_test.go index f8903aafd..c0de34250 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -424,7 +424,7 @@ func TestCluster_ResizeStates(t *testing.T) { // Before starting the resize, get the CheckSum to use for // comparison later. - node0Frame := node0.Holder.Frame("i", "f") + node0Frame := node0.Holder.Field("i", "f") node0View := node0Frame.View("standard") node0Fragment := node0View.Fragment(1) node0Checksum := node0Fragment.Checksum() @@ -453,7 +453,7 @@ func TestCluster_ResizeStates(t *testing.T) { // Bits // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. - node1Frame := node1.Holder.Frame("i", "f") + node1Frame := node1.Holder.Field("i", "f") node1View := node1Frame.View("standard") node1Fragment := node1View.Fragment(1) diff --git a/executor.go b/executor.go index b3e4f5aad..d1e40d808 100644 --- a/executor.go +++ b/executor.go @@ -369,7 +369,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq frameName, _ := c.Args["frame"].(string) - frame := e.Holder.Frame(index, frameName) + frame := e.Holder.Field(index, frameName) if frame == nil { return ValCount{}, nil } @@ -407,7 +407,7 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal frameName, _ := c.Args["frame"].(string) - frame := e.Holder.Frame(index, frameName) + frame := e.Holder.Field(index, frameName) if frame == nil { return ValCount{}, nil } @@ -445,7 +445,7 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal frameName, _ := c.Args["frame"].(string) - frame := e.Holder.Frame(index, frameName) + frame := e.Holder.Field(index, frameName) if frame == nil { return ValCount{}, nil } @@ -632,7 +632,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. if frame == "" { frame = DefaultFrame } - f := e.Holder.Frame(index, frame) + f := e.Holder.Field(index, frame) if f == nil { return nil, ErrFieldNotFound } @@ -767,7 +767,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, frameName, cond = k, vv } - f := e.Holder.Frame(index, frameName) + f := e.Holder.Field(index, frameName) if f == nil { return nil, ErrFieldNotFound } @@ -1108,7 +1108,7 @@ func (e *Executor) executeSetValue(ctx context.Context, index string, c *pql.Cal // Set values. for name, value := range args { // Retrieve frame. - frame := e.Holder.Frame(index, name) + frame := e.Holder.Field(index, name) if frame == nil { return ErrFieldNotFound } @@ -1157,7 +1157,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. } // Retrieve frame. - frame := e.Holder.Frame(index, frameName) + frame := e.Holder.Field(index, frameName) if frame == nil { return ErrFieldNotFound } @@ -1217,7 +1217,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal } // Retrieve frame. - f := e.Holder.Frame(index, frame) + f := e.Holder.Field(index, frame) if f == nil { return nil, ErrFieldNotFound } @@ -1255,7 +1255,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // Bulk insert attributes by frame. for name, frameMap := range m { // Retrieve frame. - frame := e.Holder.Frame(index, name) + frame := e.Holder.Field(index, name) if frame == nil { return nil, ErrFieldNotFound } diff --git a/executor_test.go b/executor_test.go index b8f7aa317..af4278849 100644 --- a/executor_test.go +++ b/executor_test.go @@ -289,7 +289,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Fatal(err) } - f := hldr.Frame("i", "f") + f := hldr.Field("i", "f") if value, exists, err := f.Value(10); err != nil { t.Fatal(err) } else if !exists { @@ -371,7 +371,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { t.Fatal(err) } - f := hldr.Frame("i", "f") + f := hldr.Field("i", "f") if m, err := f.RowAttrStore().Attrs(10); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { @@ -525,7 +525,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { + if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) @@ -548,7 +548,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { + if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) diff --git a/holder.go b/holder.go index f0c6d027c..f4e47e3d0 100644 --- a/holder.go +++ b/holder.go @@ -390,8 +390,8 @@ func (h *Holder) DeleteIndex(name string) error { return nil } -// Frame returns the frame for an index and name. -func (h *Holder) Frame(index, name string) *Field { +// Field returns the field for an index and name. +func (h *Holder) Field(index, name string) *Field { idx := h.Index(index) if idx == nil { return nil @@ -401,7 +401,7 @@ func (h *Holder) Frame(index, name string) *Field { // View returns the view for an index, frame, and name. func (h *Holder) View(index, frame, name string) *View { - f := h.Frame(index, frame) + f := h.Field(index, frame) if f == nil { return nil } @@ -607,7 +607,7 @@ func (s *HolderSyncer) SyncHolder() error { } // Sync frame row attributes. - if err := s.syncFrame(di.Name, fi.Name); err != nil { + if err := s.syncField(di.Name, fi.Name); err != nil { return fmt.Errorf("frame sync error: index=%s, frame=%s, err=%s", di.Name, fi.Name, err) } @@ -634,7 +634,7 @@ func (s *HolderSyncer) SyncHolder() error { } } } - s.Stats.Histogram("syncFrame", float64(time.Since(tf)), 1.0) + s.Stats.Histogram("syncField", float64(time.Since(tf)), 1.0) tf = time.Now() // reset tf } s.Stats.Histogram("syncIndex", float64(time.Since(ti)), 1.0) @@ -689,10 +689,10 @@ func (s *HolderSyncer) syncIndex(index string) error { return nil } -// syncFrame synchronizes frame attributes with the rest of the cluster. -func (s *HolderSyncer) syncFrame(index, name string) error { +// syncField synchronizes field attributes with the rest of the cluster. +func (s *HolderSyncer) syncField(index, name string) error { // Retrieve frame reference. - f := s.Holder.Frame(index, name) + f := s.Holder.Field(index, name) if f == nil { return nil } @@ -740,7 +740,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error { // syncFragment synchronizes a fragment with the rest of the cluster. func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) error { // Retrieve local frame. - f := s.Holder.Frame(index, frame) + f := s.Holder.Field(index, frame) if f == nil { return ErrFieldNotFound } diff --git a/server.go b/server.go index d92c5949e..541ff12c6 100644 --- a/server.go +++ b/server.go @@ -471,7 +471,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return err } case *internal.CreateViewMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) + f := s.Holder.Field(obj.Index, obj.Frame) if f == nil { return fmt.Errorf("Local Frame not found: %s", obj.Frame) } @@ -480,7 +480,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return err } case *internal.DeleteViewMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) + f := s.Holder.Field(obj.Index, obj.Frame) if f == nil { return fmt.Errorf("Local Frame not found: %s", obj.Frame) } diff --git a/stats_test.go b/stats_test.go index 992055166..7f71cfabb 100644 --- a/stats_test.go +++ b/stats_test.go @@ -155,7 +155,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { called := false e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - frame := e.Holder.Frame("d", "f") + frame := e.Holder.Field("d", "f") if frame == nil { t.Fatal("frame not found") } diff --git a/test/cluster.go b/test/cluster.go index e293764ad..26ef44d30 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -137,7 +137,7 @@ func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x * if c == nil { continue } - f := c.Holder.Frame(index, frame) + f := c.Holder.Field(index, frame) if f == nil { return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) } @@ -384,7 +384,7 @@ func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) if destFragment == nil { // Create fragment on destination if it doesn't exist. - f := destCluster.Holder.Frame(src.Index, src.Frame) + f := destCluster.Holder.Field(src.Index, src.Frame) v := f.View(src.View) var err error destFragment, err = v.CreateFragmentIfNotExists(src.Slice) diff --git a/utils_test.go b/utils_test.go index ccb39ef54..34b4f2967 100644 --- a/utils_test.go +++ b/utils_test.go @@ -128,7 +128,7 @@ func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, if c == nil { continue } - f := c.Holder.Frame(index, frame) + f := c.Holder.Field(index, frame) if f == nil { return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) } @@ -373,7 +373,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) if destFragment == nil { // Create fragment on destination if it doesn't exist. - f := destCluster.Holder.Frame(src.Index, src.Frame) + f := destCluster.Holder.Field(src.Index, src.Frame) v := f.View(src.View) var err error destFragment, err = v.CreateFragmentIfNotExists(src.Slice) From bcec20525b91ccb197f469859078b17b8f605122 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 22:38:52 -0500 Subject: [PATCH 38/78] GoRename Frame to Field in view.go --- view.go | 18 +++++++++--------- view_test.go | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/view.go b/view.go index 38c76a162..d76f33d59 100644 --- a/view.go +++ b/view.go @@ -39,18 +39,18 @@ func IsValidView(name string) bool { return name == ViewStandard } -// View represents a container for frame data. +// View represents a container for field data. type View struct { mu sync.RWMutex path string index string - frame string + field string name string cacheSize uint32 // Fragments by slice. - cacheType string // passed in by frame + cacheType string // passed in by field fragments map[uint64]*Fragment // maxSlice maintains this view's max slice in order to @@ -65,11 +65,11 @@ type View struct { } // NewView returns a new instance of View. -func NewView(path, index, frame, name string, cacheSize uint32) *View { +func NewView(path, index, field, name string, cacheSize uint32) *View { return &View{ path: path, index: index, - frame: frame, + field: field, name: name, cacheSize: cacheSize, @@ -88,8 +88,8 @@ func (v *View) Name() string { return v.name } // Index returns the index name the view was initialized with. func (v *View) Index() string { return v.index } -// Frame returns the frame name the view was initialized with. -func (v *View) Frame() string { return v.frame } +// Field returns the field name the view was initialized with. +func (v *View) Field() string { return v.field } // Path returns the path the view was initialized with. func (v *View) Path() string { return v.path } @@ -265,7 +265,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } func (v *View) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, v.index, v.frame, v.name, slice) + frag := NewFragment(path, v.index, v.field, v.name, slice) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize frag.Logger = v.Logger @@ -281,7 +281,7 @@ func (v *View) DeleteFragment(slice uint64) error { return ErrFragmentNotFound } - v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice) + v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, slice) // Close data files before deletion. if err := fragment.Close(); err != nil { diff --git a/view_test.go b/view_test.go index 21834113d..5bef29ddd 100644 --- a/view_test.go +++ b/view_test.go @@ -67,7 +67,7 @@ func (v *View) Reopen() error { return err } - v.View = pilosa.NewView(path, v.Index(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) + v.View = pilosa.NewView(path, v.Index(), v.Field(), v.Name(), pilosa.DefaultCacheSize) v.View.RowAttrStore = v.RowAttrStore return v.Open() } From bbf3e529dc479cce0b16206b2be8c8586a3068a4 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 22:42:21 -0500 Subject: [PATCH 39/78] GoRename Frame to Field in cluster.go --- cluster.go | 36 ++++++++++++++++++------------------ cluster_internal_test.go | 6 +++--- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/cluster.go b/cluster.go index a8f3573c5..9a467c284 100644 --- a/cluster.go +++ b/cluster.go @@ -584,7 +584,7 @@ func (c *Cluster) removeNodeBasicSorted(node *Node) bool { // frag is a struct of basic fragment information. type frag struct { - frame string + field string view string slice uint64 } @@ -617,36 +617,36 @@ func (a fragsByHost) add(b fragsByHost) fragsByHost { return a } -type viewsByFrame map[string][]string +type viewsByField map[string][]string -func (a viewsByFrame) addView(frame, view string) { - a[frame] = append(a[frame], view) +func (a viewsByField) addView(field, view string) { + a[field] = append(a[field], view) } func (c *Cluster) fragsByHost(idx *Index) fragsByHost { - // frameViews is a map of frame to slice of views. - frameViews := make(viewsByFrame) + // fieldViews is a map of field to slice of views. + fieldViews := make(viewsByField) - for _, frame := range idx.Fields() { - for _, view := range frame.Views() { - frameViews.addView(frame.Name(), view.Name()) + for _, field := range idx.Fields() { + for _, view := range field.Views() { + fieldViews.addView(field.Name(), view.Name()) } } - return c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews) + return c.fragCombos(idx.Name(), idx.MaxSlice(), fieldViews) } // fragCombos returns a map (by uri) of lists of fragments for a given index -// by creating every combination of frame/view specified in `frameViews` up to maxSlice. -func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFrame) fragsByHost { +// by creating every combination of field/view specified in `fieldViews` up to maxSlice. +func (c *Cluster) fragCombos(idx string, maxSlice uint64, fieldViews viewsByField) fragsByHost { t := make(fragsByHost) for i := uint64(0); i <= maxSlice; i++ { nodes := c.SliceNodes(idx, i) for _, n := range nodes { - // for each frame/view combination: - for frame, views := range frameViews { + // for each field/view combination: + for field, views := range fieldViews { for _, view := range views { - t[n.ID] = append(t[n.ID], frag{frame, view, i}) + t[n.ID] = append(t[n.ID], frag{field, view, i}) } } } @@ -770,7 +770,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R src := &internal.ResizeSource{ Node: EncodeNode(c.nodeByID(srcNodeID)), Index: idx.Name(), - Frame: frag.frame, + Frame: frag.field, View: frag.view, Slice: frag.slice, } @@ -1239,7 +1239,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err srcURI := decodeURI(src.Node.URI) - // Retrieve frame. + // Retrieve field. f := c.Holder.Field(src.Index, src.Frame) if f == nil { return ErrFieldNotFound @@ -1275,7 +1275,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.Node.URI) } - // Write to local frame and always close reader. + // Write to local field and always close reader. if err := func() error { defer rd.Close() _, err := frag.ReadFrom(rd) diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 9453fc16b..4b3a159f8 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -45,13 +45,13 @@ func TestFragCombos(t *testing.T) { tests := []struct { idx string maxSlice uint64 - frameViews viewsByFrame + frameViews viewsByField expected fragsByHost }{ { idx: "i", maxSlice: uint64(2), - frameViews: viewsByFrame{"f": []string{"v1", "v2"}}, + frameViews: viewsByField{"f": []string{"v1", "v2"}}, expected: fragsByHost{ "node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, "node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}}, @@ -60,7 +60,7 @@ func TestFragCombos(t *testing.T) { { idx: "foo", maxSlice: uint64(3), - frameViews: viewsByFrame{"f": []string{"v0"}}, + frameViews: viewsByField{"f": []string{"v0"}}, expected: fragsByHost{ "node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, "node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}}, From 7bb6d2b7893a0d626972512824154cfaa145ca04 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 22:45:15 -0500 Subject: [PATCH 40/78] GoRename Frame to Field in fragment.go --- fragment.go | 30 +++++++++++++++--------------- test/fragment.go | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/fragment.go b/fragment.go index dd0f05ed2..94232bfa0 100644 --- a/fragment.go +++ b/fragment.go @@ -66,13 +66,13 @@ const ( DefaultFragmentMaxOpN = 2000 ) -// Fragment represents the intersection of a frame and slice in an index. +// Fragment represents the intersection of a field and slice in an index. type Fragment struct { mu sync.RWMutex // Composite identifiers index string - frame string + field string view string slice uint64 @@ -84,7 +84,7 @@ type Fragment struct { opN int // number of ops since snapshot // Cache for row counts. - CacheType string // passed in by frame + CacheType string // passed in by field cache Cache CacheSize uint32 @@ -106,18 +106,18 @@ type Fragment struct { Logger Logger // Row attribute storage. - // This is set by the parent frame unless overridden for testing. + // This is set by the parent field unless overridden for testing. RowAttrStore AttrStore stats StatsClient } // NewFragment returns a new instance of Fragment. -func NewFragment(path, index, frame, view string, slice uint64) *Fragment { +func NewFragment(path, index, field, view string, slice uint64) *Fragment { return &Fragment{ path: path, index: index, - frame: frame, + field: field, view: view, slice: slice, CacheType: DefaultCacheType, @@ -139,8 +139,8 @@ func (f *Fragment) CachePath() string { return f.path + CacheExt } // Index returns the index that the fragment was initialized with. func (f *Fragment) Index() string { return f.index } -// Frame returns the frame the fragment was initialized with. -func (f *Fragment) Frame() string { return f.frame } +// Field returns the field the fragment was initialized with. +func (f *Fragment) Field() string { return f.field } // View returns the view the fragment was initialized with. func (f *Fragment) View() string { return f.view } @@ -247,7 +247,7 @@ func (f *Fragment) openStorage() error { // openCache initializes the cache from row ids persisted to disk. func (f *Fragment) openCache() error { - // Determine cache type from frame name. + // Determine cache type from field name. switch f.CacheType { case CacheTypeRanked: f.cache = NewRankCache(f.CacheSize) @@ -1452,8 +1452,8 @@ func track(start time.Time, message string, stats StatsClient, logger Logger) { } func (f *Fragment) snapshot() error { - f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) - completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) + f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.slice) + completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.slice) start := time.Now() defer track(start, completeMessage, f.stats, f.Logger) @@ -1783,7 +1783,7 @@ func (s *FragmentSyncer) SyncFragment() error { // Retrieve remote blocks. client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.Slice()) + blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Field(), s.Fragment.Slice()) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } @@ -1862,7 +1862,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { clients = append(clients, client) // Only sync the standard block. - rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), f.Slice(), id) + rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Field(), f.Slice(), id) if err != nil { return errors.Wrap(err, "getting block") } @@ -1904,11 +1904,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Only sync the standard block. for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Field(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) count++ } for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Field(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) count++ } diff --git a/test/fragment.go b/test/fragment.go index cd1111750..2e97d6f5d 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -76,7 +76,7 @@ func (f *Fragment) Reopen() error { return err } - f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice()) + f.Fragment = pilosa.NewFragment(path, f.Index(), f.Field(), f.View(), f.Slice()) f.Fragment.CacheType = cacheType f.Fragment.RowAttrStore = f.RowAttrStore if err := f.Open(); err != nil { From b88ebfb75445c2df7f7aa391a1b1c2a5473929d3 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 22:52:57 -0500 Subject: [PATCH 41/78] GoRename Frame to Field in diagnostics.go --- diagnostics.go | 12 ++++++------ server/server_test.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/diagnostics.go b/diagnostics.go index f97f5f6aa..e4067c6e3 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -217,7 +217,7 @@ func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { // EnrichWithSchemaProperties adds schema info to the diagnostics payload. func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { var numSlices uint64 - numFrames := 0 + numFields := 0 numIndexes := 0 bsiFieldCount := 0 timeQuantumEnabled := false @@ -225,19 +225,19 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { for _, index := range d.server.Holder.Indexes() { numSlices += index.MaxSlice() + 1 numIndexes += 1 - for _, frame := range index.Fields() { - numFrames += 1 - if frame.Type() == FieldTypeInt { + for _, field := range index.Fields() { + numFields += 1 + if field.Type() == FieldTypeInt { bsiFieldCount += 1 } - if frame.TimeQuantum() != "" { + if field.TimeQuantum() != "" { timeQuantumEnabled = true } } } d.Set("NumIndexes", numIndexes) - d.Set("NumFrames", numFrames) + d.Set("NumFields", numFields) d.Set("NumSlices", numSlices) d.Set("BSIFieldCount", bsiFieldCount) d.Set("TimeQuantumEnabled", timeQuantumEnabled) diff --git a/server/server_test.go b/server/server_test.go index 920fde809..0fa45619f 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -51,10 +51,10 @@ func TestMain_Set_Quick(t *testing.T) { // Execute SetBit() commands. for _, cmd := range cmds { - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && !strings.Contains(err.Error(), "index already exists") { + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) } - if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FieldOptions{}); err != nil && !strings.Contains(err.Error(), "frame already exists") { + if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FieldOptions{}); err != nil && err != pilosa.ErrFieldExists { t.Fatal(err) } if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { From 6dbe80350f0064671dc2db107c648664ad546bd3 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 23:02:15 -0500 Subject: [PATCH 42/78] GoRename Frame to Field in client.go --- client.go | 92 +++++++++++++++++++++++++-------------------------- ctl/import.go | 2 +- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/client.go b/client.go index d186746c7..e293c14af 100644 --- a/client.go +++ b/client.go @@ -107,7 +107,7 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]ui return rsp.Standard, nil } -// Schema returns all index and frame schema information. +// Schema returns all index and field schema information. func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) { // Execute request against the host. u := c.defaultURI.Path("/schema") @@ -269,14 +269,14 @@ func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index stri } // Import bulk imports bits for a single slice to a host. -func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error { +func (c *InternalHTTPClient) Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error { if index == "" { return ErrIndexRequired - } else if frame == "" { + } else if field == "" { return ErrFieldRequired } - buf, err := marshalImportPayload(index, frame, slice, bits) + buf, err := marshalImportPayload(index, field, slice, bits) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -298,14 +298,14 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl } // ImportK bulk imports bits specified by string keys to a host. -func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, columns []Bit) error { +func (c *InternalHTTPClient) ImportK(ctx context.Context, index, field string, columns []Bit) error { if index == "" { return ErrIndexRequired - } else if frame == "" { + } else if field == "" { return ErrFieldRequired } - buf, err := marshalImportPayloadK(index, frame, columns) + buf, err := marshalImportPayloadK(index, field, columns) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -330,8 +330,8 @@ func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, optio return err } -func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, frameName string, options FieldOptions) error { - err := c.CreateFrame(ctx, indexName, frameName, options) +func (c *InternalHTTPClient) EnsureField(ctx context.Context, indexName string, fieldName string, options FieldOptions) error { + err := c.CreateField(ctx, indexName, fieldName, options) if err == nil || err == ErrFieldExists { return nil } @@ -339,7 +339,7 @@ func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, } // marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) { +func marshalImportPayload(index, field string, slice uint64, bits []Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowIDs := Bits(bits).RowIDs() columnIDs := Bits(bits).ColumnIDs() @@ -348,7 +348,7 @@ func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, - Frame: frame, + Frame: field, Slice: slice, RowIDs: rowIDs, ColumnIDs: columnIDs, @@ -361,7 +361,7 @@ func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte } // marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. -func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) { +func marshalImportPayloadK(index, field string, bits []Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowKeys := Bits(bits).RowKeys() columnKeys := Bits(bits).ColumnKeys() @@ -370,7 +370,7 @@ func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) { // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, - Frame: frame, + Frame: field, RowKeys: rowKeys, ColumnKeys: columnKeys, Timestamps: timestamps, @@ -420,14 +420,14 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []b } // ImportValue bulk imports field values for a single slice to a host. -func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame string, slice uint64, vals []FieldValue) error { +func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error { if index == "" { return ErrIndexRequired - } else if frame == "" { + } else if field == "" { return ErrFieldRequired } - buf, err := marshalImportValuePayload(index, frame, slice, vals) + buf, err := marshalImportValuePayload(index, field, slice, vals) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -449,7 +449,7 @@ func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame strin } // marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func marshalImportValuePayload(index, frame string, slice uint64, vals []FieldValue) ([]byte, error) { +func marshalImportValuePayload(index, field string, slice uint64, vals []FieldValue) ([]byte, error) { // Separate row and column IDs to reduce allocations. columnIDs := FieldValues(vals).ColumnIDs() values := FieldValues(vals).Values() @@ -457,7 +457,7 @@ func marshalImportValuePayload(index, frame string, slice uint64, vals []FieldVa // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportValueRequest{ Index: index, - Frame: frame, + Frame: field, Slice: slice, ColumnIDs: columnIDs, Values: values, @@ -507,10 +507,10 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu } // ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error { +func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { if index == "" { return ErrIndexRequired - } else if frame == "" { + } else if field == "" { return ErrFieldRequired } @@ -525,7 +525,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame string, for _, i := range rand.Perm(len(nodes)) { node := nodes[i] - if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil { + if err := c.exportNodeCSV(ctx, node, index, field, slice, w); err != nil { e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) continue } else { @@ -537,12 +537,12 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame string, } // exportNode copies a CSV export from a node to w. -func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error { +func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, field string, slice uint64, w io.Writer) error { // Create URL. u := nodePathToURL(node, "/export") u.RawQuery = url.Values{ "index": {index}, - "frame": {frame}, + "frame": {field}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode() @@ -574,18 +574,18 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde return nil } -func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame string, slice uint64, uri URI) (io.ReadCloser, error) { +func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) { node := &Node{ URI: uri, } - return c.backupSliceNode(ctx, index, frame, slice, node) + return c.backupSliceNode(ctx, index, field, slice, node) } -func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame string, slice uint64, node *Node) (io.ReadCloser, error) { +func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *Node) (io.ReadCloser, error) { u := nodePathToURL(node, "/fragment/data") u.RawQuery = url.Values{ "index": {index}, - "frame": {frame}, + "frame": {field}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode() @@ -615,8 +615,8 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame s return resp.Body, nil } -// CreateFrame creates a new frame on the server. -func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FieldOptions) error { +// CreateField creates a new field on the server. +func (c *InternalHTTPClient) CreateField(ctx context.Context, index, field string, opt FieldOptions) error { if index == "" { return ErrIndexRequired } @@ -630,7 +630,7 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin } // Create URL & HTTP request. - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, frame)) + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, field)) req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { return errors.Wrap(err, "creating request") @@ -666,11 +666,11 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) { +func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, field string, slice uint64) ([]FragmentBlock, error) { u := uriPathToURL(c.defaultURI, "/fragment/blocks") u.RawQuery = url.Values{ "index": {index}, - "frame": {frame}, + "frame": {field}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode() @@ -707,10 +707,10 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame st } // BlockData returns row/column id pairs for a block. -func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error) { +func (c *InternalHTTPClient) BlockData(ctx context.Context, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ Index: index, - Frame: frame, + Frame: field, Slice: slice, Block: uint64(block), }) @@ -794,8 +794,8 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b } // RowAttrDiff returns data from differing blocks on a remote host. -func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame)) +func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, field)) // Encode request. buf, err := json.Marshal(postFrameAttrDiffRequest{Blocks: blks}) @@ -961,7 +961,7 @@ func (p Bits) GroupBySlice() map[uint64][]Bit { } // FieldValues represents the value for a column within a -// range-encoded frame. +// range-encoded field. type FieldValue struct { ColumnID uint64 Value int64 @@ -1053,16 +1053,16 @@ type InternalClient interface { FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) - Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error - ImportK(ctx context.Context, index, frame string, bits []Bit) error + Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error + ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error - EnsureFrame(ctx context.Context, indexName string, frameName string, options FieldOptions) error - ImportValue(ctx context.Context, index, frame string, slice uint64, vals []FieldValue) error - ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error - CreateFrame(ctx context.Context, index, frame string, opt FieldOptions) error - FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error) + EnsureField(ctx context.Context, indexName string, fieldName string, options FieldOptions) error + ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error + ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error + CreateField(ctx context.Context, index, field string, opt FieldOptions) error + FragmentBlocks(ctx context.Context, index, field string, slice uint64) ([]FragmentBlock, error) + BlockData(ctx context.Context, index, field string, slice uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + RowAttrDiff(ctx context.Context, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error } diff --git a/ctl/import.go b/ctl/import.go index 68229ed72..0bc3d093c 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -134,7 +134,7 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { if err != nil { return fmt.Errorf("Error Creating Index: %s", err) } - err = cmd.Client.EnsureFrame(ctx, cmd.Index, cmd.Frame, cmd.FrameOptions) + err = cmd.Client.EnsureField(ctx, cmd.Index, cmd.Frame, cmd.FrameOptions) if err != nil { return fmt.Errorf("Error Creating Frame: %s", err) } From 4254eb1d1140d732772e64fe9b66bd6d6b560812 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 23:10:59 -0500 Subject: [PATCH 43/78] rename internal FrameMeta to FieldOptions --- frame.go | 10 +-- holder_test.go | 2 +- internal/private.pb.go | 187 +++++++++++++++++++++-------------------- internal/private.proto | 6 +- 4 files changed, 103 insertions(+), 102 deletions(-) diff --git a/frame.go b/frame.go index b69848344..c830213f7 100644 --- a/frame.go +++ b/frame.go @@ -261,7 +261,7 @@ func (f *Field) openViews() error { // loadMeta reads meta data for the frame, if any. func (f *Field) loadMeta() error { - var pb internal.FrameMeta + var pb internal.FieldOptions // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) @@ -1038,15 +1038,15 @@ func (o *FieldOptions) Validate() error { } // Encode converts o into its internal representation. -func (o *FieldOptions) Encode() *internal.FrameMeta { +func (o *FieldOptions) Encode() *internal.FieldOptions { return encodeFieldOptions(o) } -func encodeFieldOptions(o *FieldOptions) *internal.FrameMeta { +func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { if o == nil { return nil } - return &internal.FrameMeta{ + return &internal.FieldOptions{ Type: o.Type, CacheType: o.CacheType, CacheSize: o.CacheSize, @@ -1056,7 +1056,7 @@ func encodeFieldOptions(o *FieldOptions) *internal.FrameMeta { } } -func decodeFieldOptions(options *internal.FrameMeta) *FieldOptions { +func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions { if options == nil { return nil } diff --git a/holder_test.go b/holder_test.go index 664bf16a0..35e3b8e26 100644 --- a/holder_test.go +++ b/holder_test.go @@ -113,7 +113,7 @@ func TestHolder_Open(t *testing.T) { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrFrameMetaCorrupt", func(t *testing.T) { + t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) { h := test.MustOpenHolder() defer h.Close() diff --git a/internal/private.pb.go b/internal/private.pb.go index a4f9e0431..56c8eba81 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -10,7 +10,7 @@ It has these top-level messages: IndexMeta - FrameMeta + FieldOptions ImportResponse BlockDataRequest BlockDataResponse @@ -68,7 +68,7 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } -type FrameMeta struct { +type FieldOptions struct { Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` @@ -77,47 +77,47 @@ type FrameMeta struct { TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } -func (m *FrameMeta) Reset() { *m = FrameMeta{} } -func (m *FrameMeta) String() string { return proto.CompactTextString(m) } -func (*FrameMeta) ProtoMessage() {} -func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } +func (m *FieldOptions) Reset() { *m = FieldOptions{} } +func (m *FieldOptions) String() string { return proto.CompactTextString(m) } +func (*FieldOptions) ProtoMessage() {} +func (*FieldOptions) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } -func (m *FrameMeta) GetType() string { +func (m *FieldOptions) GetType() string { if m != nil { return m.Type } return "" } -func (m *FrameMeta) GetCacheType() string { +func (m *FieldOptions) GetCacheType() string { if m != nil { return m.CacheType } return "" } -func (m *FrameMeta) GetCacheSize() uint32 { +func (m *FieldOptions) GetCacheSize() uint32 { if m != nil { return m.CacheSize } return 0 } -func (m *FrameMeta) GetMin() int64 { +func (m *FieldOptions) GetMin() int64 { if m != nil { return m.Min } return 0 } -func (m *FrameMeta) GetMax() int64 { +func (m *FieldOptions) GetMax() int64 { if m != nil { return m.Max } return 0 } -func (m *FrameMeta) GetTimeQuantum() string { +func (m *FieldOptions) GetTimeQuantum() string { if m != nil { return m.TimeQuantum } @@ -309,9 +309,9 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { } type CreateFrameMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Meta *FrameMeta `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` } func (m *CreateFrameMessage) Reset() { *m = CreateFrameMessage{} } @@ -333,7 +333,7 @@ func (m *CreateFrameMessage) GetFrame() string { return "" } -func (m *CreateFrameMessage) GetMeta() *FrameMeta { +func (m *CreateFrameMessage) GetMeta() *FieldOptions { if m != nil { return m.Meta } @@ -365,9 +365,9 @@ func (m *DeleteFrameMessage) GetFrame() string { } type Frame struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` } func (m *Frame) Reset() { *m = Frame{} } @@ -382,7 +382,7 @@ func (m *Frame) GetName() string { return "" } -func (m *Frame) GetMeta() *FrameMeta { +func (m *Frame) GetMeta() *FieldOptions { if m != nil { return m.Meta } @@ -920,7 +920,7 @@ func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPr func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") - proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") + proto.RegisterType((*FieldOptions)(nil), "internal.FieldOptions") proto.RegisterType((*ImportResponse)(nil), "internal.ImportResponse") proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest") proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") @@ -969,7 +969,7 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *FrameMeta) Marshal() (dAtA []byte, err error) { +func (m *FieldOptions) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -979,7 +979,7 @@ func (m *FrameMeta) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { +func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -2138,7 +2138,7 @@ func (m *IndexMeta) Size() (n int) { return n } -func (m *FrameMeta) Size() (n int) { +func (m *FieldOptions) Size() (n int) { var l int _ = l l = len(m.CacheType) @@ -2696,7 +2696,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { } return nil } -func (m *FrameMeta) Unmarshal(dAtA []byte) error { +func (m *FieldOptions) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -2719,10 +2719,10 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: FrameMeta: wiretype end group for non-group") + return fmt.Errorf("proto: FieldOptions: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: FieldOptions: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 3: @@ -3989,7 +3989,7 @@ func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Meta == nil { - m.Meta = &FrameMeta{} + m.Meta = &FieldOptions{} } if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -4209,7 +4209,7 @@ func (m *Frame) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Meta == nil { - m.Meta = &FrameMeta{} + m.Meta = &FieldOptions{} } if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -6617,68 +6617,69 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1002 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcd, 0x6f, 0x1b, 0x45, - 0x14, 0x67, 0xbd, 0x6b, 0xc7, 0x7e, 0xc1, 0xc1, 0x99, 0x42, 0xd8, 0x22, 0x14, 0xcc, 0xa8, 0x12, - 0x86, 0x43, 0x54, 0xda, 0x0b, 0x5f, 0x95, 0xa2, 0xd8, 0x01, 0x16, 0x91, 0x08, 0x66, 0x93, 0x1e, - 0x90, 0x38, 0x4c, 0xed, 0x51, 0xbb, 0xca, 0x7a, 0xc7, 0xec, 0xce, 0x26, 0x71, 0x0f, 0x5c, 0xe1, - 0xc2, 0x1d, 0x71, 0xe3, 0xbf, 0xe1, 0xc8, 0x9f, 0x80, 0xc2, 0x3f, 0x82, 0xe6, 0xcd, 0xec, 0x47, - 0xfc, 0xd1, 0x54, 0xa1, 0xb7, 0x79, 0xbf, 0xf7, 0xfd, 0xde, 0x6f, 0x76, 0x16, 0xba, 0xb3, 0x34, - 0x3a, 0xe7, 0x4a, 0xec, 0xcd, 0x52, 0xa9, 0x24, 0x69, 0x47, 0x89, 0x12, 0x69, 0xc2, 0x63, 0xba, - 0x09, 0x9d, 0x20, 0x99, 0x88, 0xcb, 0x23, 0xa1, 0x38, 0xfd, 0xd3, 0x81, 0xce, 0x97, 0x29, 0x9f, - 0x0a, 0x2d, 0x91, 0x77, 0xa1, 0x33, 0xe4, 0xe3, 0x67, 0xe2, 0x64, 0x3e, 0x13, 0xbe, 0xdb, 0x77, - 0x06, 0x1d, 0x56, 0x01, 0xa5, 0x36, 0x8c, 0x9e, 0x0b, 0xdf, 0xeb, 0x3b, 0x83, 0x2e, 0xab, 0x00, - 0xd2, 0x87, 0xcd, 0x93, 0x68, 0x2a, 0xbe, 0xcf, 0x79, 0xa2, 0xf2, 0xa9, 0xdf, 0x44, 0xef, 0x3a, - 0x44, 0x08, 0x78, 0x18, 0xb8, 0x8d, 0x2a, 0x3c, 0x93, 0x1e, 0xb8, 0x47, 0x51, 0xe2, 0x77, 0xfa, - 0xce, 0xc0, 0x65, 0xfa, 0x88, 0x08, 0xbf, 0xf4, 0xc1, 0x22, 0xfc, 0x92, 0x52, 0xd8, 0x0a, 0xa6, - 0x33, 0x99, 0x2a, 0x26, 0xb2, 0x99, 0x4c, 0x32, 0xf4, 0x3a, 0x4c, 0x53, 0xdf, 0xc1, 0x40, 0xfa, - 0x48, 0x7f, 0x86, 0xde, 0x41, 0x2c, 0xc7, 0x67, 0x23, 0xae, 0x38, 0x13, 0x3f, 0xe5, 0x22, 0x53, - 0xe4, 0x4d, 0x68, 0x62, 0xa3, 0xd6, 0xce, 0x08, 0x1a, 0xc5, 0x86, 0xfd, 0x86, 0x41, 0x51, 0xd0, - 0x28, 0xfa, 0x63, 0xd7, 0x1e, 0x33, 0x82, 0x46, 0xc3, 0x38, 0x1a, 0x9b, 0x6e, 0x3d, 0x66, 0x04, - 0xdd, 0xc7, 0xe3, 0x48, 0x5c, 0xd8, 0x16, 0xf1, 0x4c, 0x03, 0xd8, 0xae, 0xe5, 0xb7, 0x65, 0xee, - 0x40, 0x8b, 0xc9, 0x8b, 0x60, 0x94, 0xf9, 0x4e, 0xdf, 0x1d, 0x78, 0xcc, 0x4a, 0x38, 0x48, 0x19, - 0xe7, 0xd3, 0x44, 0xab, 0x1a, 0xa8, 0xaa, 0x00, 0x7a, 0x17, 0x9a, 0x38, 0x55, 0xdd, 0x65, 0xe5, - 0xab, 0x8f, 0xf4, 0x17, 0x07, 0x3a, 0x47, 0xfc, 0x12, 0xcb, 0xc8, 0xc8, 0x23, 0x68, 0x87, 0x8a, - 0x27, 0x13, 0x9e, 0x4e, 0xd0, 0x68, 0xf3, 0xc1, 0xfb, 0x7b, 0xc5, 0x96, 0xf7, 0x4a, 0xb3, 0xbd, - 0xc2, 0xe6, 0x30, 0x51, 0xe9, 0x9c, 0x95, 0x2e, 0xef, 0x7c, 0x0e, 0xdd, 0x6b, 0x2a, 0x9d, 0xef, - 0x4c, 0xcc, 0x8b, 0xa9, 0x9e, 0x89, 0xb9, 0xee, 0xff, 0x9c, 0xc7, 0xb9, 0x99, 0x95, 0xc7, 0x8c, - 0xf0, 0x59, 0xe3, 0x13, 0x87, 0xee, 0x03, 0x19, 0xa6, 0x82, 0x2b, 0x81, 0x49, 0x8e, 0x44, 0x96, - 0xf1, 0xa7, 0x62, 0xfd, 0xc4, 0xcd, 0x14, 0x1b, 0xb5, 0x29, 0xd2, 0x8f, 0x80, 0x8c, 0x44, 0x2c, - 0x94, 0xb0, 0x64, 0x7c, 0x41, 0x04, 0x1a, 0x16, 0xd9, 0x6e, 0xb6, 0x25, 0x1f, 0x80, 0xa7, 0xb9, - 0x8c, 0xc9, 0x36, 0x1f, 0xdc, 0xa9, 0x26, 0x52, 0x92, 0x9e, 0xa1, 0x01, 0x8d, 0x8a, 0xa0, 0x96, - 0xff, 0x37, 0xb4, 0xb0, 0x82, 0x34, 0x45, 0x2a, 0x77, 0x31, 0x55, 0x79, 0xa3, 0x6c, 0xaa, 0xfd, - 0xa2, 0xd7, 0xdb, 0xa6, 0xa2, 0x3f, 0x58, 0x54, 0x93, 0xef, 0x58, 0x6b, 0x8d, 0x0f, 0x9e, 0xd7, - 0xb7, 0xbc, 0x50, 0x87, 0x8e, 0xad, 0xd9, 0x9a, 0xf9, 0x6e, 0xdf, 0xd5, 0xb1, 0x51, 0xa0, 0x0f, - 0xa1, 0x15, 0x8e, 0x9f, 0x89, 0x29, 0x27, 0x1f, 0xc2, 0x06, 0x16, 0x21, 0x32, 0x4b, 0xa8, 0x37, - 0x16, 0xc6, 0xc7, 0x0a, 0x3d, 0x1d, 0xd9, 0xe2, 0xd7, 0x14, 0xd4, 0xc2, 0xd4, 0x99, 0xef, 0x2d, - 0x86, 0x41, 0x9c, 0x59, 0x35, 0x3d, 0x04, 0xf7, 0x94, 0x05, 0xfa, 0xa2, 0x60, 0x05, 0x45, 0x14, - 0x2b, 0xe9, 0xd8, 0x5f, 0xcb, 0x4c, 0xd9, 0x51, 0xe0, 0x59, 0x63, 0xdf, 0xc9, 0x54, 0xe1, 0xd0, - 0xbb, 0x0c, 0xcf, 0xf4, 0x47, 0xf0, 0x8e, 0xe5, 0x44, 0x90, 0x2d, 0x68, 0x04, 0x23, 0x1b, 0xa3, - 0x11, 0x8c, 0xc8, 0x7b, 0x18, 0xde, 0xce, 0xa5, 0x5b, 0x15, 0x71, 0xca, 0x02, 0x86, 0x89, 0xef, - 0x41, 0x37, 0xc8, 0x86, 0x52, 0xa6, 0x93, 0x28, 0xe1, 0x4a, 0xa6, 0x18, 0xb5, 0xcd, 0xae, 0x83, - 0x74, 0x1f, 0x7a, 0x3a, 0x7c, 0xa8, 0xb8, 0x2a, 0x97, 0xb7, 0x03, 0x2d, 0x8d, 0x95, 0xe9, 0xac, - 0x84, 0x64, 0xd7, 0x76, 0xc5, 0xfa, 0x50, 0xa0, 0xdf, 0x9a, 0x08, 0x87, 0xe7, 0x22, 0x51, 0xb5, - 0xf5, 0xa3, 0x8c, 0x01, 0xba, 0xcc, 0x08, 0x84, 0x9a, 0x56, 0x6c, 0xcd, 0x5b, 0x55, 0xcd, 0x1a, - 0x65, 0xa8, 0xa3, 0xbf, 0x39, 0x00, 0x45, 0x41, 0x79, 0x56, 0xba, 0x38, 0xeb, 0x5d, 0xc8, 0xc7, - 0xb5, 0x0f, 0xc7, 0x32, 0x4f, 0x4a, 0x15, 0xab, 0x7d, 0x5e, 0x06, 0x05, 0x2d, 0x2c, 0xbf, 0x7b, - 0x95, 0xbd, 0xc1, 0xed, 0x9a, 0xf4, 0x4d, 0xea, 0x0e, 0xe3, 0x3c, 0x53, 0x22, 0xb5, 0x15, 0xe9, - 0x0f, 0x9c, 0x01, 0xca, 0xf9, 0x54, 0xc0, 0xea, 0x11, 0x91, 0x7b, 0xd0, 0xd4, 0x95, 0x1a, 0x6e, - 0x2e, 0xb7, 0x61, 0x94, 0xf4, 0x31, 0xb4, 0x0f, 0xc2, 0xe0, 0xab, 0x54, 0xe6, 0xb3, 0x95, 0xcc, - 0x2b, 0xde, 0x98, 0xc6, 0xf2, 0x1b, 0xe3, 0x2e, 0xbd, 0x31, 0x5e, 0xf5, 0xc6, 0x84, 0xb0, 0x6d, - 0x3e, 0x06, 0xfa, 0x4a, 0xdc, 0xe6, 0x5b, 0x50, 0x3c, 0x0a, 0x6e, 0xed, 0x51, 0x08, 0x61, 0xdb, - 0x5c, 0xfb, 0x57, 0x19, 0xf4, 0x8f, 0x06, 0x6c, 0x33, 0x91, 0x45, 0xcf, 0x45, 0x90, 0x64, 0x2a, - 0xcd, 0xc7, 0x2a, 0x92, 0x89, 0xf6, 0xff, 0x46, 0x3e, 0xb1, 0xd3, 0x76, 0x99, 0x11, 0x5e, 0x86, - 0x4c, 0xe4, 0x3e, 0x6c, 0x2e, 0x5e, 0x80, 0x65, 0xd3, 0xba, 0x09, 0xb9, 0x0f, 0x1b, 0xa1, 0xcc, - 0xd3, 0x71, 0x79, 0xbd, 0x77, 0x2a, 0x6b, 0x53, 0x99, 0x51, 0xb3, 0xc2, 0xac, 0x46, 0xa5, 0xe6, - 0x8b, 0xa9, 0x44, 0x1e, 0x2d, 0x50, 0xc9, 0x6f, 0xa1, 0xc3, 0xdb, 0x95, 0xc3, 0x35, 0x35, 0xbb, - 0x6e, 0x4d, 0x7f, 0x75, 0xe0, 0xf5, 0x7a, 0x09, 0x2f, 0x75, 0x37, 0xca, 0x8d, 0x34, 0x56, 0x6e, - 0xc4, 0x5d, 0xb5, 0x11, 0xaf, 0xda, 0x48, 0xf5, 0xbe, 0x35, 0xeb, 0xef, 0xdb, 0x19, 0xdc, 0x5d, - 0x5a, 0xd3, 0x50, 0x4e, 0x67, 0x9a, 0x0f, 0xff, 0x63, 0x5d, 0xfa, 0xab, 0x91, 0xa6, 0x76, 0x51, - 0x1d, 0x66, 0x04, 0xfa, 0x29, 0xbc, 0x15, 0x0a, 0x55, 0x5b, 0x52, 0xc1, 0xb6, 0x3e, 0xb8, 0xc7, - 0xe2, 0x62, 0x4d, 0xfb, 0x5a, 0x45, 0xbf, 0x00, 0xff, 0x74, 0x36, 0xe1, 0x4a, 0xdc, 0xca, 0xfb, - 0x00, 0xda, 0x27, 0x72, 0x26, 0x63, 0xf9, 0x74, 0x7e, 0xc3, 0xad, 0xf7, 0x61, 0xc3, 0x7c, 0x22, - 0xcd, 0x2f, 0x4f, 0x87, 0x15, 0x22, 0xbd, 0xa3, 0x09, 0x3d, 0xe6, 0xf1, 0x38, 0x8f, 0x75, 0x19, - 0xfa, 0xdf, 0x27, 0x3b, 0xe8, 0xfd, 0x75, 0xb5, 0xeb, 0xfc, 0x7d, 0xb5, 0xeb, 0xfc, 0x73, 0xb5, - 0xeb, 0xfc, 0xfe, 0xef, 0xee, 0x6b, 0x4f, 0x5a, 0xf8, 0x23, 0xfb, 0xf0, 0xbf, 0x00, 0x00, 0x00, - 0xff, 0xff, 0x93, 0x15, 0x15, 0x14, 0xd9, 0x0a, 0x00, 0x00, + // 1011 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1c, 0x35, + 0x18, 0x67, 0x1e, 0xbb, 0xd9, 0xfd, 0xd2, 0x0d, 0x89, 0x0b, 0x61, 0x8a, 0x50, 0x58, 0xac, 0x4a, + 0x0d, 0x3d, 0x44, 0xa5, 0xbd, 0xf0, 0xaa, 0x14, 0x25, 0x9b, 0xc2, 0x20, 0x12, 0xc0, 0x93, 0xf4, + 0xd6, 0x83, 0xbb, 0x6b, 0xb5, 0xa3, 0xcc, 0x8e, 0x87, 0x19, 0x4f, 0x92, 0xed, 0x81, 0x2b, 0x5c, + 0xb8, 0x23, 0xce, 0xfc, 0x31, 0x1c, 0xf9, 0x13, 0x50, 0xf8, 0x47, 0x90, 0x3f, 0x7b, 0x1e, 0xc9, + 0x6e, 0x9a, 0x2a, 0x70, 0xf3, 0xf7, 0x7e, 0xfd, 0x3e, 0xdb, 0x30, 0xc8, 0xf2, 0xf8, 0x84, 0x2b, + 0xb1, 0x95, 0xe5, 0x52, 0x49, 0xd2, 0x8b, 0x53, 0x25, 0xf2, 0x94, 0x27, 0x74, 0x19, 0xfa, 0x61, + 0x3a, 0x11, 0x67, 0xfb, 0x42, 0x71, 0xfa, 0x87, 0x03, 0xb7, 0x9e, 0xc4, 0x22, 0x99, 0x7c, 0x97, + 0xa9, 0x58, 0xa6, 0x05, 0xf9, 0x00, 0xfa, 0xbb, 0x7c, 0xfc, 0x52, 0x1c, 0xce, 0x32, 0x11, 0x78, + 0x43, 0x67, 0xb3, 0xcf, 0x1a, 0x46, 0x2d, 0x8d, 0xe2, 0x57, 0x22, 0xf0, 0x87, 0xce, 0xe6, 0x80, + 0x35, 0x0c, 0x32, 0x84, 0xe5, 0xc3, 0x78, 0x2a, 0x7e, 0x28, 0x79, 0xaa, 0xca, 0x69, 0xd0, 0x41, + 0xeb, 0x36, 0x8b, 0x10, 0xf0, 0xd1, 0x71, 0x0f, 0x45, 0x78, 0x26, 0xab, 0xe0, 0xed, 0xc7, 0x69, + 0xd0, 0x1f, 0x3a, 0x9b, 0x1e, 0xd3, 0x47, 0xe4, 0xf0, 0xb3, 0x00, 0x2c, 0x87, 0x9f, 0x51, 0x0a, + 0x2b, 0xe1, 0x34, 0x93, 0xb9, 0x62, 0xa2, 0xc8, 0x64, 0x5a, 0xa0, 0xd5, 0x5e, 0x9e, 0x07, 0x0e, + 0x3a, 0xd2, 0x47, 0xfa, 0x13, 0xac, 0xee, 0x24, 0x72, 0x7c, 0x3c, 0xe2, 0x8a, 0x33, 0xf1, 0x63, + 0x29, 0x0a, 0x45, 0xde, 0x81, 0x0e, 0xd6, 0x6a, 0xf5, 0x0c, 0xa1, 0xb9, 0x4f, 0x72, 0x3e, 0x15, + 0x81, 0x6b, 0xb8, 0x48, 0x68, 0x2e, 0xda, 0x63, 0xd5, 0x3e, 0x33, 0x84, 0xe6, 0x46, 0x49, 0x3c, + 0x36, 0xd5, 0xfa, 0xcc, 0x10, 0xba, 0x8e, 0xa7, 0xb1, 0x38, 0xb5, 0x25, 0xe2, 0x99, 0x86, 0xb0, + 0xd6, 0x8a, 0x6f, 0xd3, 0x5c, 0x87, 0x2e, 0x93, 0xa7, 0xe1, 0xa8, 0x08, 0x9c, 0xa1, 0xb7, 0xe9, + 0x33, 0x4b, 0x61, 0x23, 0x65, 0x52, 0x4e, 0x53, 0x2d, 0x72, 0x51, 0xd4, 0x30, 0xe8, 0x1d, 0xe8, + 0x60, 0x57, 0x75, 0x95, 0x8d, 0xad, 0x3e, 0xd2, 0x9f, 0x1d, 0xe8, 0xef, 0xf3, 0x33, 0x4c, 0xa3, + 0x20, 0x8f, 0xa1, 0x17, 0x29, 0x9e, 0x4e, 0x78, 0x3e, 0x41, 0xa5, 0xe5, 0x87, 0x1f, 0x6d, 0x55, + 0x83, 0xde, 0xaa, 0xd5, 0xb6, 0x2a, 0x9d, 0xbd, 0x54, 0xe5, 0x33, 0x56, 0x9b, 0xbc, 0xff, 0x05, + 0x0c, 0x2e, 0x88, 0x74, 0xbc, 0x63, 0x31, 0xab, 0xba, 0x7a, 0x2c, 0x66, 0xba, 0xfe, 0x13, 0x9e, + 0x94, 0xa6, 0x57, 0x3e, 0x33, 0xc4, 0xe7, 0xee, 0xa7, 0x0e, 0xdd, 0x06, 0xb2, 0x9b, 0x0b, 0xae, + 0x04, 0x06, 0xd9, 0x17, 0x45, 0xc1, 0x5f, 0x88, 0xab, 0x3b, 0x6e, 0xba, 0xe8, 0xb6, 0xba, 0x48, + 0xef, 0x03, 0x19, 0x89, 0x44, 0x28, 0x61, 0xf1, 0xf8, 0x1a, 0x0f, 0x34, 0xaa, 0xa2, 0x5d, 0xaf, + 0x4b, 0xee, 0x81, 0xaf, 0xc1, 0x8d, 0xc1, 0x96, 0x1f, 0xde, 0x6e, 0x3a, 0x52, 0xe3, 0x9e, 0xa1, + 0x02, 0x4d, 0x2a, 0xa7, 0x88, 0x80, 0x6b, 0x4b, 0x58, 0x00, 0x9a, 0xfb, 0x36, 0x94, 0x87, 0xa1, + 0xd6, 0x9b, 0x50, 0xed, 0xa5, 0xb2, 0xd1, 0xb6, 0xab, 0x72, 0x6f, 0x1a, 0x8d, 0x3e, 0xb3, 0x5c, + 0x8d, 0xbf, 0x03, 0x2d, 0x35, 0x36, 0x78, 0xae, 0x53, 0x71, 0xaf, 0x4f, 0x45, 0xbb, 0xd7, 0x98, + 0x2d, 0x02, 0x6f, 0xe8, 0x69, 0xf7, 0x48, 0xd0, 0x47, 0xd0, 0x8d, 0xc6, 0x2f, 0xc5, 0x94, 0x93, + 0x8f, 0x61, 0x09, 0xf3, 0x10, 0x85, 0x85, 0xd5, 0xdb, 0x97, 0x9a, 0xc8, 0x2a, 0x39, 0x1d, 0xd9, + 0xfc, 0x17, 0xe6, 0x74, 0x0f, 0xba, 0x98, 0x70, 0x11, 0xf8, 0x97, 0xdd, 0x20, 0x9f, 0x59, 0x31, + 0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5d, 0x30, 0x83, 0xca, 0x8b, 0xa5, 0xb4, 0xef, 0xaf, 0x65, + 0xa1, 0x6c, 0x37, 0xf0, 0xac, 0x79, 0xdf, 0xcb, 0x5c, 0x61, 0xeb, 0x07, 0x0c, 0xcf, 0xf4, 0x19, + 0xf8, 0x07, 0x72, 0x22, 0xc8, 0x0a, 0xb8, 0xe1, 0xc8, 0xfa, 0x70, 0xc3, 0x11, 0xf9, 0x10, 0xdd, + 0xdb, 0xd6, 0x0c, 0x9a, 0x24, 0x8e, 0x58, 0xc8, 0x30, 0xf0, 0x5d, 0x18, 0x84, 0xc5, 0xae, 0x94, + 0xf9, 0x24, 0x4e, 0xb9, 0x92, 0x39, 0x7a, 0xed, 0xb1, 0x8b, 0x4c, 0xba, 0x0d, 0xab, 0xda, 0x7d, + 0xa4, 0xb8, 0xaa, 0xe7, 0xb7, 0x0e, 0x5d, 0xcd, 0xab, 0xc3, 0x59, 0x0a, 0x21, 0xaf, 0xf5, 0xaa, + 0x09, 0x22, 0x41, 0xbf, 0x35, 0x1e, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x08, 0x40, 0x1a, 0x1d, 0x0c, + 0x98, 0x21, 0x08, 0x35, 0xa5, 0xd8, 0x9c, 0x57, 0x9a, 0x9c, 0x35, 0x97, 0xa1, 0x8c, 0xfe, 0xea, + 0x00, 0x54, 0x09, 0x95, 0x45, 0x6d, 0xe2, 0x5c, 0x6d, 0x42, 0x3e, 0x69, 0x5d, 0x1f, 0xf3, 0x0b, + 0x52, 0x8b, 0x58, 0xeb, 0x92, 0xd9, 0xac, 0x60, 0x61, 0x51, 0xbe, 0xda, 0xe8, 0x1b, 0xbe, 0x1d, + 0x13, 0xa7, 0x31, 0x0c, 0x76, 0x93, 0xb2, 0x50, 0x22, 0xb7, 0x19, 0xe9, 0x6b, 0xce, 0x30, 0xea, + 0xfe, 0x34, 0x8c, 0xc5, 0x2d, 0x22, 0x77, 0xa1, 0xa3, 0x33, 0x35, 0xd8, 0x9c, 0x2f, 0xc3, 0x08, + 0xe9, 0x53, 0xe8, 0xed, 0x44, 0xe1, 0x57, 0xb9, 0x2c, 0xb3, 0x85, 0xc8, 0xab, 0x5e, 0x1a, 0x77, + 0xfe, 0xa5, 0xf1, 0xe6, 0x5e, 0x1a, 0xbf, 0x79, 0x69, 0x22, 0x58, 0x33, 0x57, 0x82, 0x5e, 0x89, + 0x9b, 0xdc, 0x08, 0xd5, 0xd3, 0xe0, 0xb5, 0x9e, 0x86, 0x08, 0xd6, 0xcc, 0xe6, 0xff, 0x9f, 0x4e, + 0x7f, 0x77, 0x61, 0x8d, 0x89, 0x22, 0x7e, 0x25, 0xc2, 0xb4, 0x50, 0x79, 0x39, 0xd6, 0x0b, 0xae, + 0xed, 0xbf, 0x91, 0xcf, 0x6d, 0xb7, 0x3d, 0x66, 0x88, 0x37, 0x01, 0x13, 0x79, 0x00, 0xcb, 0x97, + 0x17, 0x60, 0x5e, 0xb5, 0xad, 0x42, 0x1e, 0xc0, 0x52, 0x24, 0xcb, 0x7c, 0x5c, 0xaf, 0x77, 0xeb, + 0xd2, 0x31, 0x99, 0x19, 0x31, 0xab, 0xd4, 0x5a, 0x50, 0xea, 0xbc, 0x1e, 0x4a, 0xe4, 0xf1, 0x25, + 0x28, 0x05, 0x5d, 0x34, 0x78, 0xaf, 0x31, 0xb8, 0x20, 0x66, 0x17, 0xb5, 0xe9, 0x2f, 0x0e, 0xdc, + 0x6a, 0xa7, 0xf0, 0x46, 0xbb, 0x51, 0x4f, 0xc4, 0x5d, 0x38, 0x11, 0x6f, 0xd1, 0x44, 0xfc, 0x66, + 0x22, 0xcd, 0x2b, 0xd7, 0x69, 0xbf, 0x72, 0xc7, 0x70, 0x67, 0x6e, 0x4c, 0xbb, 0x72, 0x9a, 0x69, + 0x3c, 0xfc, 0x87, 0x71, 0xe9, 0x5b, 0x23, 0xcf, 0xed, 0xa0, 0xfa, 0xcc, 0x10, 0xf4, 0x33, 0x78, + 0x37, 0x12, 0xaa, 0x35, 0xa4, 0x0a, 0x6d, 0x43, 0xf0, 0x0e, 0xc4, 0xe9, 0x15, 0xe5, 0x6b, 0x11, + 0xfd, 0x12, 0x82, 0xa3, 0x6c, 0xc2, 0x95, 0xb8, 0x91, 0xf5, 0x0e, 0xf4, 0x0e, 0x65, 0x26, 0x13, + 0xf9, 0x62, 0x76, 0xcd, 0xd6, 0x07, 0xb0, 0x64, 0xae, 0x48, 0xf3, 0xf1, 0xe9, 0xb3, 0x8a, 0xa4, + 0xb7, 0x35, 0xa0, 0xc7, 0x3c, 0x19, 0x97, 0x89, 0x4e, 0x43, 0xff, 0x80, 0x8a, 0x9d, 0xd5, 0x3f, + 0xcf, 0x37, 0x9c, 0xbf, 0xce, 0x37, 0x9c, 0xbf, 0xcf, 0x37, 0x9c, 0xdf, 0xfe, 0xd9, 0x78, 0xeb, + 0x79, 0x17, 0x7f, 0xb4, 0x8f, 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xe6, 0x4d, 0xbe, 0x70, 0xe2, + 0x0a, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index d150b7802..bf846f01c 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -5,7 +5,7 @@ package internal; message IndexMeta { } -message FrameMeta { +message FieldOptions { string Type = 8; string CacheType = 3; uint32 CacheSize = 4; @@ -56,7 +56,7 @@ message CreateIndexMessage { message CreateFrameMessage { string Index = 1; string Frame = 2; - FrameMeta Meta = 3; + FieldOptions Meta = 3; } message DeleteFrameMessage { @@ -66,7 +66,7 @@ message DeleteFrameMessage { message Frame { string Name = 1; - FrameMeta Meta = 2; + FieldOptions Meta = 2; repeated string Views = 3; } From 636bf252adbe57f9b476087ac78bda15488d4695 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 23:13:49 -0500 Subject: [PATCH 44/78] rename internal Frame to Field --- frame.go | 8 +-- internal/private.pb.go | 110 ++++++++++++++++++++--------------------- internal/private.proto | 4 +- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/frame.go b/frame.go index c830213f7..1cd94cc83 100644 --- a/frame.go +++ b/frame.go @@ -971,8 +971,8 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { } // encodeFields converts a into its internal representation. -func encodeFields(a []*Field) []*internal.Frame { - other := make([]*internal.Frame, len(a)) +func encodeFields(a []*Field) []*internal.Field { + other := make([]*internal.Field, len(a)) for i := range a { other[i] = encodeField(a[i]) } @@ -980,9 +980,9 @@ func encodeFields(a []*Field) []*internal.Frame { } // encodeField converts f into its internal representation. -func encodeField(f *Field) *internal.Frame { +func encodeField(f *Field) *internal.Field { fo := f.options - return &internal.Frame{ + return &internal.Field{ Name: f.name, Meta: fo.Encode(), Views: f.viewNames(), diff --git a/internal/private.pb.go b/internal/private.pb.go index 56c8eba81..ebeda0e6b 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -21,7 +21,7 @@ CreateIndexMessage CreateFrameMessage DeleteFrameMessage - Frame + Field Schema Index URI @@ -364,32 +364,32 @@ func (m *DeleteFrameMessage) GetFrame() string { return "" } -type Frame struct { +type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` } -func (m *Frame) Reset() { *m = Frame{} } -func (m *Frame) String() string { return proto.CompactTextString(m) } -func (*Frame) ProtoMessage() {} -func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } +func (m *Field) Reset() { *m = Field{} } +func (m *Field) String() string { return proto.CompactTextString(m) } +func (*Field) ProtoMessage() {} +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } -func (m *Frame) GetName() string { +func (m *Field) GetName() string { if m != nil { return m.Name } return "" } -func (m *Frame) GetMeta() *FieldOptions { +func (m *Field) GetMeta() *FieldOptions { if m != nil { return m.Meta } return nil } -func (m *Frame) GetViews() []string { +func (m *Field) GetViews() []string { if m != nil { return m.Views } @@ -414,7 +414,7 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` + Frames []*Field `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` } func (m *Index) Reset() { *m = Index{} } @@ -429,7 +429,7 @@ func (m *Index) GetName() string { return "" } -func (m *Index) GetFrames() []*Frame { +func (m *Index) GetFrames() []*Field { if m != nil { return m.Frames } @@ -931,7 +931,7 @@ func init() { proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") - proto.RegisterType((*Frame)(nil), "internal.Frame") + proto.RegisterType((*Field)(nil), "internal.Field") proto.RegisterType((*Schema)(nil), "internal.Schema") proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*URI)(nil), "internal.URI") @@ -1368,7 +1368,7 @@ func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Frame) Marshal() (dAtA []byte, err error) { +func (m *Field) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1378,7 +1378,7 @@ func (m *Frame) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Frame) MarshalTo(dAtA []byte) (int, error) { +func (m *Field) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -2315,7 +2315,7 @@ func (m *DeleteFrameMessage) Size() (n int) { return n } -func (m *Frame) Size() (n int) { +func (m *Field) Size() (n int) { var l int _ = l l = len(m.Name) @@ -4124,7 +4124,7 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *Frame) Unmarshal(dAtA []byte) error { +func (m *Field) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -4147,10 +4147,10 @@ func (m *Frame) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Frame: wiretype end group for non-group") + return fmt.Errorf("proto: Field: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Frame: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Field: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -4430,7 +4430,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frames = append(m.Frames, &Frame{}) + m.Frames = append(m.Frames, &Field{}) if err := m.Frames[len(m.Frames)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } @@ -6617,7 +6617,7 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1011 bytes of a gzipped FileDescriptorProto + // 1013 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1c, 0x35, 0x18, 0x67, 0x1e, 0xbb, 0xd9, 0xfd, 0xd2, 0x0d, 0x89, 0x0b, 0x61, 0x8a, 0x50, 0x58, 0xac, 0x4a, 0x0d, 0x3d, 0x44, 0xa5, 0xbd, 0xf0, 0xaa, 0x14, 0x25, 0x9b, 0xc2, 0x20, 0x12, 0xc0, 0x93, 0xf4, @@ -6647,39 +6647,39 @@ var fileDescriptorPrivate = []byte{ 0xef, 0x03, 0x19, 0x89, 0x44, 0x28, 0x61, 0xf1, 0xf8, 0x1a, 0x0f, 0x34, 0xaa, 0xa2, 0x5d, 0xaf, 0x4b, 0xee, 0x81, 0xaf, 0xc1, 0x8d, 0xc1, 0x96, 0x1f, 0xde, 0x6e, 0x3a, 0x52, 0xe3, 0x9e, 0xa1, 0x02, 0x4d, 0x2a, 0xa7, 0x88, 0x80, 0x6b, 0x4b, 0x58, 0x00, 0x9a, 0xfb, 0x36, 0x94, 0x87, 0xa1, - 0xd6, 0x9b, 0x50, 0xed, 0xa5, 0xb2, 0xd1, 0xb6, 0xab, 0x72, 0x6f, 0x1a, 0x8d, 0x3e, 0xb3, 0x5c, - 0x8d, 0xbf, 0x03, 0x2d, 0x35, 0x36, 0x78, 0xae, 0x53, 0x71, 0xaf, 0x4f, 0x45, 0xbb, 0xd7, 0x98, - 0x2d, 0x02, 0x6f, 0xe8, 0x69, 0xf7, 0x48, 0xd0, 0x47, 0xd0, 0x8d, 0xc6, 0x2f, 0xc5, 0x94, 0x93, - 0x8f, 0x61, 0x09, 0xf3, 0x10, 0x85, 0x85, 0xd5, 0xdb, 0x97, 0x9a, 0xc8, 0x2a, 0x39, 0x1d, 0xd9, - 0xfc, 0x17, 0xe6, 0x74, 0x0f, 0xba, 0x98, 0x70, 0x11, 0xf8, 0x97, 0xdd, 0x20, 0x9f, 0x59, 0x31, - 0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5d, 0x30, 0x83, 0xca, 0x8b, 0xa5, 0xb4, 0xef, 0xaf, 0x65, - 0xa1, 0x6c, 0x37, 0xf0, 0xac, 0x79, 0xdf, 0xcb, 0x5c, 0x61, 0xeb, 0x07, 0x0c, 0xcf, 0xf4, 0x19, - 0xf8, 0x07, 0x72, 0x22, 0xc8, 0x0a, 0xb8, 0xe1, 0xc8, 0xfa, 0x70, 0xc3, 0x11, 0xf9, 0x10, 0xdd, - 0xdb, 0xd6, 0x0c, 0x9a, 0x24, 0x8e, 0x58, 0xc8, 0x30, 0xf0, 0x5d, 0x18, 0x84, 0xc5, 0xae, 0x94, - 0xf9, 0x24, 0x4e, 0xb9, 0x92, 0x39, 0x7a, 0xed, 0xb1, 0x8b, 0x4c, 0xba, 0x0d, 0xab, 0xda, 0x7d, - 0xa4, 0xb8, 0xaa, 0xe7, 0xb7, 0x0e, 0x5d, 0xcd, 0xab, 0xc3, 0x59, 0x0a, 0x21, 0xaf, 0xf5, 0xaa, - 0x09, 0x22, 0x41, 0xbf, 0x35, 0x1e, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x08, 0x40, 0x1a, 0x1d, 0x0c, - 0x98, 0x21, 0x08, 0x35, 0xa5, 0xd8, 0x9c, 0x57, 0x9a, 0x9c, 0x35, 0x97, 0xa1, 0x8c, 0xfe, 0xea, - 0x00, 0x54, 0x09, 0x95, 0x45, 0x6d, 0xe2, 0x5c, 0x6d, 0x42, 0x3e, 0x69, 0x5d, 0x1f, 0xf3, 0x0b, - 0x52, 0x8b, 0x58, 0xeb, 0x92, 0xd9, 0xac, 0x60, 0x61, 0x51, 0xbe, 0xda, 0xe8, 0x1b, 0xbe, 0x1d, - 0x13, 0xa7, 0x31, 0x0c, 0x76, 0x93, 0xb2, 0x50, 0x22, 0xb7, 0x19, 0xe9, 0x6b, 0xce, 0x30, 0xea, - 0xfe, 0x34, 0x8c, 0xc5, 0x2d, 0x22, 0x77, 0xa1, 0xa3, 0x33, 0x35, 0xd8, 0x9c, 0x2f, 0xc3, 0x08, - 0xe9, 0x53, 0xe8, 0xed, 0x44, 0xe1, 0x57, 0xb9, 0x2c, 0xb3, 0x85, 0xc8, 0xab, 0x5e, 0x1a, 0x77, - 0xfe, 0xa5, 0xf1, 0xe6, 0x5e, 0x1a, 0xbf, 0x79, 0x69, 0x22, 0x58, 0x33, 0x57, 0x82, 0x5e, 0x89, - 0x9b, 0xdc, 0x08, 0xd5, 0xd3, 0xe0, 0xb5, 0x9e, 0x86, 0x08, 0xd6, 0xcc, 0xe6, 0xff, 0x9f, 0x4e, - 0x7f, 0x77, 0x61, 0x8d, 0x89, 0x22, 0x7e, 0x25, 0xc2, 0xb4, 0x50, 0x79, 0x39, 0xd6, 0x0b, 0xae, - 0xed, 0xbf, 0x91, 0xcf, 0x6d, 0xb7, 0x3d, 0x66, 0x88, 0x37, 0x01, 0x13, 0x79, 0x00, 0xcb, 0x97, - 0x17, 0x60, 0x5e, 0xb5, 0xad, 0x42, 0x1e, 0xc0, 0x52, 0x24, 0xcb, 0x7c, 0x5c, 0xaf, 0x77, 0xeb, - 0xd2, 0x31, 0x99, 0x19, 0x31, 0xab, 0xd4, 0x5a, 0x50, 0xea, 0xbc, 0x1e, 0x4a, 0xe4, 0xf1, 0x25, - 0x28, 0x05, 0x5d, 0x34, 0x78, 0xaf, 0x31, 0xb8, 0x20, 0x66, 0x17, 0xb5, 0xe9, 0x2f, 0x0e, 0xdc, - 0x6a, 0xa7, 0xf0, 0x46, 0xbb, 0x51, 0x4f, 0xc4, 0x5d, 0x38, 0x11, 0x6f, 0xd1, 0x44, 0xfc, 0x66, - 0x22, 0xcd, 0x2b, 0xd7, 0x69, 0xbf, 0x72, 0xc7, 0x70, 0x67, 0x6e, 0x4c, 0xbb, 0x72, 0x9a, 0x69, - 0x3c, 0xfc, 0x87, 0x71, 0xe9, 0x5b, 0x23, 0xcf, 0xed, 0xa0, 0xfa, 0xcc, 0x10, 0xf4, 0x33, 0x78, - 0x37, 0x12, 0xaa, 0x35, 0xa4, 0x0a, 0x6d, 0x43, 0xf0, 0x0e, 0xc4, 0xe9, 0x15, 0xe5, 0x6b, 0x11, - 0xfd, 0x12, 0x82, 0xa3, 0x6c, 0xc2, 0x95, 0xb8, 0x91, 0xf5, 0x0e, 0xf4, 0x0e, 0x65, 0x26, 0x13, - 0xf9, 0x62, 0x76, 0xcd, 0xd6, 0x07, 0xb0, 0x64, 0xae, 0x48, 0xf3, 0xf1, 0xe9, 0xb3, 0x8a, 0xa4, - 0xb7, 0x35, 0xa0, 0xc7, 0x3c, 0x19, 0x97, 0x89, 0x4e, 0x43, 0xff, 0x80, 0x8a, 0x9d, 0xd5, 0x3f, - 0xcf, 0x37, 0x9c, 0xbf, 0xce, 0x37, 0x9c, 0xbf, 0xcf, 0x37, 0x9c, 0xdf, 0xfe, 0xd9, 0x78, 0xeb, - 0x79, 0x17, 0x7f, 0xb4, 0x8f, 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xe6, 0x4d, 0xbe, 0x70, 0xe2, - 0x0a, 0x00, 0x00, + 0xd6, 0x9b, 0x50, 0xed, 0xa5, 0xb2, 0xd1, 0xb6, 0xab, 0x72, 0x6f, 0x1a, 0x8d, 0x3e, 0x83, 0x0e, + 0xfa, 0xd5, 0xf8, 0x3b, 0xd0, 0x52, 0x63, 0x83, 0xe7, 0x3a, 0x15, 0xf7, 0xfa, 0x54, 0xb4, 0x7b, + 0x8d, 0xd9, 0x22, 0xf0, 0x86, 0x9e, 0x76, 0x8f, 0x04, 0x7d, 0x04, 0xdd, 0x68, 0xfc, 0x52, 0x4c, + 0x39, 0xf9, 0x18, 0x96, 0x30, 0x0f, 0x51, 0x58, 0x58, 0xbd, 0x7d, 0xa9, 0x89, 0xac, 0x92, 0xd3, + 0x91, 0xcd, 0x7f, 0x61, 0x4e, 0xf7, 0xa0, 0x8b, 0x99, 0x17, 0x81, 0x7f, 0xd9, 0x0d, 0x66, 0xc5, + 0xac, 0x98, 0xee, 0x81, 0x77, 0xc4, 0x42, 0xbd, 0x2e, 0x98, 0x41, 0xe5, 0xc5, 0x52, 0xda, 0xf7, + 0xd7, 0xb2, 0x50, 0xb6, 0x1b, 0x78, 0xd6, 0xbc, 0xef, 0x65, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x67, + 0xfa, 0x0c, 0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x7d, 0xb8, 0xe1, 0x88, 0x7c, + 0x88, 0xee, 0x6d, 0x6b, 0x06, 0x4d, 0x12, 0x47, 0x2c, 0x64, 0x18, 0xf8, 0x2e, 0x0c, 0xc2, 0x62, + 0x57, 0xca, 0x7c, 0x12, 0xa7, 0x5c, 0xc9, 0x1c, 0xbd, 0xf6, 0xd8, 0x45, 0x26, 0xdd, 0x86, 0x55, + 0xed, 0x3e, 0x52, 0x5c, 0xd5, 0xf3, 0x5b, 0x87, 0xae, 0xe6, 0xd5, 0xe1, 0x2c, 0x85, 0x90, 0xd7, + 0x7a, 0xd5, 0x04, 0x91, 0xa0, 0xdf, 0x1a, 0x0f, 0x7b, 0x27, 0x22, 0x55, 0x2d, 0x04, 0x20, 0x8d, + 0x0e, 0x06, 0xcc, 0x10, 0x84, 0x9a, 0x52, 0x6c, 0xce, 0x2b, 0x4d, 0xce, 0x9a, 0xcb, 0x50, 0x46, + 0x7f, 0x75, 0x00, 0xaa, 0x84, 0xca, 0xa2, 0x36, 0x71, 0xae, 0x36, 0x21, 0x9f, 0xb4, 0xae, 0x8f, + 0xf9, 0x05, 0xa9, 0x45, 0xac, 0x75, 0xc9, 0x6c, 0x56, 0xb0, 0xb0, 0x28, 0x5f, 0x6d, 0xf4, 0x0d, + 0xdf, 0x8e, 0x89, 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0xdb, 0x8c, 0xf4, 0x35, 0x67, + 0x18, 0x75, 0x7f, 0x1a, 0xc6, 0xe2, 0x16, 0x91, 0xbb, 0xd0, 0xd1, 0x99, 0x1a, 0x6c, 0xce, 0x97, + 0x61, 0x84, 0xf4, 0x29, 0xf4, 0x76, 0xa2, 0xf0, 0xab, 0x5c, 0x96, 0xd9, 0x42, 0xe4, 0x55, 0x2f, + 0x8d, 0x3b, 0xff, 0xd2, 0x78, 0x73, 0x2f, 0x8d, 0xdf, 0xbc, 0x34, 0x11, 0xac, 0x99, 0x2b, 0x41, + 0xaf, 0xc4, 0x4d, 0x6e, 0x84, 0xea, 0x69, 0xf0, 0x5a, 0x4f, 0x43, 0x04, 0x6b, 0x66, 0xf3, 0xff, + 0x4f, 0xa7, 0xbf, 0xbb, 0xb0, 0xc6, 0x44, 0x11, 0xbf, 0x12, 0x61, 0x5a, 0xa8, 0xbc, 0x1c, 0xeb, + 0x05, 0xd7, 0xf6, 0xdf, 0xc8, 0xe7, 0xb6, 0xdb, 0x1e, 0x33, 0xc4, 0x9b, 0x80, 0x89, 0x3c, 0x80, + 0xe5, 0xcb, 0x0b, 0x30, 0xaf, 0xda, 0x56, 0x21, 0x0f, 0x60, 0x29, 0x92, 0x65, 0x3e, 0xae, 0xd7, + 0xbb, 0x75, 0xe9, 0x98, 0xcc, 0x8c, 0x98, 0x55, 0x6a, 0x2d, 0x28, 0x75, 0x5e, 0x0f, 0x25, 0xf2, + 0xf8, 0x12, 0x94, 0x82, 0x2e, 0x1a, 0xbc, 0xd7, 0x18, 0x5c, 0x10, 0xb3, 0x8b, 0xda, 0xf4, 0x17, + 0x07, 0x6e, 0xb5, 0x53, 0x78, 0xa3, 0xdd, 0xa8, 0x27, 0xe2, 0x2e, 0x9c, 0x88, 0xb7, 0x68, 0x22, + 0x7e, 0x33, 0x91, 0xe6, 0x95, 0xeb, 0xb4, 0x5f, 0xb9, 0x63, 0xb8, 0x33, 0x37, 0xa6, 0x5d, 0x39, + 0xcd, 0x34, 0x1e, 0xfe, 0xc3, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0x76, 0x50, 0x7d, 0x66, 0x08, 0xfa, + 0x19, 0xbc, 0x1b, 0x09, 0xd5, 0x1a, 0x52, 0x85, 0xb6, 0x21, 0x78, 0x07, 0xe2, 0xf4, 0x8a, 0xf2, + 0xb5, 0x88, 0x7e, 0x09, 0xc1, 0x51, 0x36, 0xe1, 0x4a, 0xdc, 0xc8, 0x7a, 0x07, 0x7a, 0x87, 0x32, + 0x93, 0x89, 0x7c, 0x31, 0xbb, 0x66, 0xeb, 0x03, 0x58, 0x32, 0x57, 0xa4, 0xf9, 0xf8, 0xf4, 0x59, + 0x45, 0xd2, 0xdb, 0x1a, 0xd0, 0x63, 0x9e, 0x8c, 0xcb, 0x44, 0xa7, 0xa1, 0x7f, 0x40, 0xc5, 0xce, + 0xea, 0x9f, 0xe7, 0x1b, 0xce, 0x5f, 0xe7, 0x1b, 0xce, 0xdf, 0xe7, 0x1b, 0xce, 0x6f, 0xff, 0x6c, + 0xbc, 0xf5, 0xbc, 0x8b, 0x3f, 0xda, 0x47, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xab, 0x64, 0x15, + 0x6e, 0xe2, 0x0a, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index bf846f01c..30d41205b 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -64,7 +64,7 @@ message DeleteFrameMessage { string Frame = 2; } -message Frame { +message Field { string Name = 1; FieldOptions Meta = 2; repeated string Views = 3; @@ -76,7 +76,7 @@ message Schema { message Index { string Name = 1; - repeated Frame Frames = 4; + repeated Field Frames = 4; } message URI { From cb7487e34d534735ad1395705339bb70a7160586 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 23:24:24 -0500 Subject: [PATCH 45/78] change all instances of internal Frame to Field --- api.go | 32 ++--- broadcast.go | 20 +-- client.go | 8 +- cluster.go | 6 +- frame.go | 2 +- holder.go | 2 +- index.go | 2 +- internal/private.pb.go | 310 ++++++++++++++++++++--------------------- internal/private.proto | 18 +-- internal/public.pb.go | 126 ++++++++--------- internal/public.proto | 4 +- server.go | 16 +-- 12 files changed, 273 insertions(+), 273 deletions(-) diff --git a/api.go b/api.go index 9bd3abdf4..ae010e4f1 100644 --- a/api.go +++ b/api.go @@ -237,16 +237,16 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str // Send the create frame message to all nodes. err = api.Broadcaster.SendSync( - &internal.CreateFrameMessage{ + &internal.CreateFieldMessage{ Index: indexName, - Frame: frameName, + Field: frameName, Meta: options.Encode(), }) if err != nil { - api.Logger.Printf("problem sending CreateFrame message: %s", err) - return nil, errors.Wrap(err, "sending CreateFrame message") + api.Logger.Printf("problem sending CreateField message: %s", err) + return nil, errors.Wrap(err, "sending CreateField message") } - api.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + api.Holder.Stats.CountWithCustomTags("createField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) return frame, nil } @@ -271,15 +271,15 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str // Send the delete frame message to all nodes. err := api.Broadcaster.SendSync( - &internal.DeleteFrameMessage{ + &internal.DeleteFieldMessage{ Index: indexName, - Frame: frameName, + Field: frameName, }) if err != nil { - api.Logger.Printf("problem sending DeleteFrame message: %s", err) - return errors.Wrap(err, "sending DeleteFrame message") + api.Logger.Printf("problem sending DeleteField message: %s", err) + return errors.Wrap(err, "sending DeleteField message") } - api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + api.Holder.Stats.CountWithCustomTags("deleteField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) return nil } @@ -397,7 +397,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } // Retrieve fragment from holder. - f := api.Holder.Fragment(req.Index, req.Frame, ViewStandard, req.Slice) + f := api.Holder.Fragment(req.Index, req.Field, ViewStandard, req.Slice) if f == nil { return nil, ErrFragmentNotFound } @@ -529,7 +529,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri err := api.Broadcaster.SendSync( &internal.DeleteViewMessage{ Index: indexName, - Frame: frameName, + Field: frameName, View: viewName, }) if err != nil { @@ -614,7 +614,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { return errors.Wrap(err, "validating api method") } - _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + _, frame, err := api.indexFrame(req.Index, req.Field, req.Slice) if err != nil { return errors.Wrap(err, "getting frame") } @@ -632,7 +632,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // Import into fragment. err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } @@ -643,7 +643,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest return errors.Wrap(err, "validating api method") } - _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + _, frame, err := api.indexFrame(req.Index, req.Field, req.Slice) if err != nil { return errors.Wrap(err, "getting frame") } @@ -651,7 +651,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // Import into fragment. err = frame.ImportValue(req.ColumnIDs, req.Values) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } diff --git a/broadcast.go b/broadcast.go index 933ca05a5..6b26b5d58 100644 --- a/broadcast.go +++ b/broadcast.go @@ -123,8 +123,8 @@ const ( MessageTypeCreateSlice = iota MessageTypeCreateIndex MessageTypeDeleteIndex - MessageTypeCreateFrame - MessageTypeDeleteFrame + MessageTypeCreateField + MessageTypeDeleteField MessageTypeCreateView MessageTypeDeleteView MessageTypeClusterStatus @@ -147,10 +147,10 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeCreateIndex case *internal.DeleteIndexMessage: typ = MessageTypeDeleteIndex - case *internal.CreateFrameMessage: - typ = MessageTypeCreateFrame - case *internal.DeleteFrameMessage: - typ = MessageTypeDeleteFrame + case *internal.CreateFieldMessage: + typ = MessageTypeCreateField + case *internal.DeleteFieldMessage: + typ = MessageTypeDeleteField case *internal.CreateViewMessage: typ = MessageTypeCreateView case *internal.DeleteViewMessage: @@ -193,10 +193,10 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.CreateIndexMessage{} case MessageTypeDeleteIndex: m = &internal.DeleteIndexMessage{} - case MessageTypeCreateFrame: - m = &internal.CreateFrameMessage{} - case MessageTypeDeleteFrame: - m = &internal.DeleteFrameMessage{} + case MessageTypeCreateField: + m = &internal.CreateFieldMessage{} + case MessageTypeDeleteField: + m = &internal.DeleteFieldMessage{} case MessageTypeCreateView: m = &internal.CreateViewMessage{} case MessageTypeDeleteView: diff --git a/client.go b/client.go index e293c14af..ae536f2c4 100644 --- a/client.go +++ b/client.go @@ -348,7 +348,7 @@ func marshalImportPayload(index, field string, slice uint64, bits []Bit) ([]byte // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, - Frame: field, + Field: field, Slice: slice, RowIDs: rowIDs, ColumnIDs: columnIDs, @@ -370,7 +370,7 @@ func marshalImportPayloadK(index, field string, bits []Bit) ([]byte, error) { // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, - Frame: field, + Field: field, RowKeys: rowKeys, ColumnKeys: columnKeys, Timestamps: timestamps, @@ -457,7 +457,7 @@ func marshalImportValuePayload(index, field string, slice uint64, vals []FieldVa // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportValueRequest{ Index: index, - Frame: field, + Field: field, Slice: slice, ColumnIDs: columnIDs, Values: values, @@ -710,7 +710,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, field st func (c *InternalHTTPClient) BlockData(ctx context.Context, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ Index: index, - Frame: field, + Field: field, Slice: slice, Block: uint64(block), }) diff --git a/cluster.go b/cluster.go index 9a467c284..9723e5bce 100644 --- a/cluster.go +++ b/cluster.go @@ -770,7 +770,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R src := &internal.ResizeSource{ Node: EncodeNode(c.nodeByID(srcNodeID)), Index: idx.Name(), - Frame: frag.field, + Field: frag.field, View: frag.view, Slice: frag.slice, } @@ -1240,7 +1240,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err srcURI := decodeURI(src.Node.URI) // Retrieve field. - f := c.Holder.Field(src.Index, src.Frame) + f := c.Holder.Field(src.Index, src.Field) if f == nil { return ErrFieldNotFound } @@ -1259,7 +1259,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Stream slice from remote node. c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) - rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.Slice, srcURI) + rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.Slice, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found // on the remote node. This occurs when a slice has been skipped and diff --git a/frame.go b/frame.go index 1cd94cc83..60e41e265 100644 --- a/frame.go +++ b/frame.go @@ -556,7 +556,7 @@ func (f *Field) CreateViewIfNotExists(name string) (*View, error) { err = f.broadcaster.SendSync( &internal.CreateViewMessage{ Index: f.index, - Frame: f.name, + Field: f.name, View: name, }) if err != nil { diff --git a/holder.go b/holder.go index f4e47e3d0..bbff69927 100644 --- a/holder.go +++ b/holder.go @@ -239,7 +239,7 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { return errors.Wrap(err, "creating index") } // Create frames that don't exist. - for _, f := range index.Frames { + for _, f := range index.Fields { opt := decodeFieldOptions(f.Meta) frame, err := idx.CreateFieldIfNotExists(f.Name, *opt) if err != nil { diff --git a/index.go b/index.go index 727989914..72b9cc1e7 100644 --- a/index.go +++ b/index.go @@ -402,7 +402,7 @@ func EncodeIndexes(a []*Index) []*internal.Index { func encodeIndex(d *Index) *internal.Index { return &internal.Index{ Name: d.name, - Frames: encodeFields(d.Fields()), + Fields: encodeFields(d.Fields()), } } diff --git a/internal/private.pb.go b/internal/private.pb.go index ebeda0e6b..1e2b34f38 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -19,8 +19,8 @@ CreateSliceMessage DeleteIndexMessage CreateIndexMessage - CreateFrameMessage - DeleteFrameMessage + CreateFieldMessage + DeleteFieldMessage Field Schema Index @@ -142,7 +142,7 @@ func (m *ImportResponse) GetErr() string { type BlockDataRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` Slice uint64 `protobuf:"varint,4,opt,name=Slice,proto3" json:"Slice,omitempty"` Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"` @@ -160,9 +160,9 @@ func (m *BlockDataRequest) GetIndex() string { return "" } -func (m *BlockDataRequest) GetFrame() string { +func (m *BlockDataRequest) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -308,58 +308,58 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { return nil } -type CreateFrameMessage struct { +type CreateFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` } -func (m *CreateFrameMessage) Reset() { *m = CreateFrameMessage{} } -func (m *CreateFrameMessage) String() string { return proto.CompactTextString(m) } -func (*CreateFrameMessage) ProtoMessage() {} -func (*CreateFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } +func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } +func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } +func (*CreateFieldMessage) ProtoMessage() {} +func (*CreateFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } -func (m *CreateFrameMessage) GetIndex() string { +func (m *CreateFieldMessage) GetIndex() string { if m != nil { return m.Index } return "" } -func (m *CreateFrameMessage) GetFrame() string { +func (m *CreateFieldMessage) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } -func (m *CreateFrameMessage) GetMeta() *FieldOptions { +func (m *CreateFieldMessage) GetMeta() *FieldOptions { if m != nil { return m.Meta } return nil } -type DeleteFrameMessage struct { +type DeleteFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` } -func (m *DeleteFrameMessage) Reset() { *m = DeleteFrameMessage{} } -func (m *DeleteFrameMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteFrameMessage) ProtoMessage() {} -func (*DeleteFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } +func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } +func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteFieldMessage) ProtoMessage() {} +func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } -func (m *DeleteFrameMessage) GetIndex() string { +func (m *DeleteFieldMessage) GetIndex() string { if m != nil { return m.Index } return "" } -func (m *DeleteFrameMessage) GetFrame() string { +func (m *DeleteFieldMessage) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -414,7 +414,7 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Frames []*Field `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` + Fields []*Field `protobuf:"bytes,4,rep,name=Fields" json:"Fields,omitempty"` } func (m *Index) Reset() { *m = Index{} } @@ -429,9 +429,9 @@ func (m *Index) GetName() string { return "" } -func (m *Index) GetFrames() []*Field { +func (m *Index) GetFields() []*Field { if m != nil { - return m.Frames + return m.Fields } return nil } @@ -654,7 +654,7 @@ func (m *BSIGroup) GetMax() int64 { type CreateViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` } @@ -670,9 +670,9 @@ func (m *CreateViewMessage) GetIndex() string { return "" } -func (m *CreateViewMessage) GetFrame() string { +func (m *CreateViewMessage) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -686,7 +686,7 @@ func (m *CreateViewMessage) GetView() string { type DeleteViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` } @@ -702,9 +702,9 @@ func (m *DeleteViewMessage) GetIndex() string { return "" } -func (m *DeleteViewMessage) GetFrame() string { +func (m *DeleteViewMessage) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -775,7 +775,7 @@ func (m *ResizeInstruction) GetClusterStatus() *ClusterStatus { type ResizeSource struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,3,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` Slice uint64 `protobuf:"varint,5,opt,name=Slice,proto3" json:"Slice,omitempty"` } @@ -799,9 +799,9 @@ func (m *ResizeSource) GetIndex() string { return "" } -func (m *ResizeSource) GetFrame() string { +func (m *ResizeSource) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -929,8 +929,8 @@ func init() { proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage") proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage") proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") - proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") - proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") + proto.RegisterType((*CreateFieldMessage)(nil), "internal.CreateFieldMessage") + proto.RegisterType((*DeleteFieldMessage)(nil), "internal.DeleteFieldMessage") proto.RegisterType((*Field)(nil), "internal.Field") proto.RegisterType((*Schema)(nil), "internal.Schema") proto.RegisterType((*Index)(nil), "internal.Index") @@ -1065,11 +1065,11 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if m.Block != 0 { dAtA[i] = 0x18 @@ -1298,7 +1298,7 @@ func (m *CreateIndexMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *CreateFrameMessage) Marshal() (dAtA []byte, err error) { +func (m *CreateFieldMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1308,7 +1308,7 @@ func (m *CreateFrameMessage) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *CreateFrameMessage) MarshalTo(dAtA []byte) (int, error) { +func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1319,11 +1319,11 @@ func (m *CreateFrameMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if m.Meta != nil { dAtA[i] = 0x1a @@ -1338,7 +1338,7 @@ func (m *CreateFrameMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *DeleteFrameMessage) Marshal() (dAtA []byte, err error) { +func (m *DeleteFieldMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1348,7 +1348,7 @@ func (m *DeleteFrameMessage) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { +func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1359,11 +1359,11 @@ func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } return i, nil } @@ -1468,8 +1468,8 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) i += copy(dAtA[i:], m.Name) } - if len(m.Frames) > 0 { - for _, msg := range m.Frames { + if len(m.Fields) > 0 { + for _, msg := range m.Fields { dAtA[i] = 0x22 i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) @@ -1776,11 +1776,11 @@ func (m *CreateViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if len(m.View) > 0 { dAtA[i] = 0x1a @@ -1812,11 +1812,11 @@ func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if len(m.View) > 0 { dAtA[i] = 0x1a @@ -1933,11 +1933,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x1a i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if len(m.View) > 0 { dAtA[i] = 0x22 @@ -2182,7 +2182,7 @@ func (m *BlockDataRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2283,14 +2283,14 @@ func (m *CreateIndexMessage) Size() (n int) { return n } -func (m *CreateFrameMessage) Size() (n int) { +func (m *CreateFieldMessage) Size() (n int) { var l int _ = l l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2301,14 +2301,14 @@ func (m *CreateFrameMessage) Size() (n int) { return n } -func (m *DeleteFrameMessage) Size() (n int) { +func (m *DeleteFieldMessage) Size() (n int) { var l int _ = l l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2354,8 +2354,8 @@ func (m *Index) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if len(m.Frames) > 0 { - for _, e := range m.Frames { + if len(m.Fields) > 0 { + for _, e := range m.Fields { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } @@ -2489,7 +2489,7 @@ func (m *CreateViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2507,7 +2507,7 @@ func (m *DeleteViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2560,7 +2560,7 @@ func (m *ResizeSource) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -3029,7 +3029,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3054,7 +3054,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 0 { @@ -3875,7 +3875,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { +func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3898,10 +3898,10 @@ func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: CreateFrameMessage: wiretype end group for non-group") + return fmt.Errorf("proto: CreateFieldMessage: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: CreateFrameMessage: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: CreateFieldMessage: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -3935,7 +3935,7 @@ func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3960,7 +3960,7 @@ func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { @@ -4016,7 +4016,7 @@ func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { +func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -4039,10 +4039,10 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: DeleteFrameMessage: wiretype end group for non-group") + return fmt.Errorf("proto: DeleteFieldMessage: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: DeleteFrameMessage: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: DeleteFieldMessage: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -4076,7 +4076,7 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -4101,7 +4101,7 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex @@ -4406,7 +4406,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frames", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -4430,8 +4430,8 @@ func (m *Index) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frames = append(m.Frames, &Field{}) - if err := m.Frames[len(m.Frames)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Fields = append(m.Fields, &Field{}) + if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -5419,7 +5419,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -5444,7 +5444,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { @@ -5556,7 +5556,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -5581,7 +5581,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { @@ -5958,7 +5958,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -5983,7 +5983,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 4: if wireType != 2 { @@ -6617,69 +6617,69 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1013 bytes of a gzipped FileDescriptorProto + // 1011 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1c, 0x35, 0x18, 0x67, 0x1e, 0xbb, 0xd9, 0xfd, 0xd2, 0x0d, 0x89, 0x0b, 0x61, 0x8a, 0x50, 0x58, 0xac, 0x4a, - 0x0d, 0x3d, 0x44, 0xa5, 0xbd, 0xf0, 0xaa, 0x14, 0x25, 0x9b, 0xc2, 0x20, 0x12, 0xc0, 0x93, 0xf4, - 0xd6, 0x83, 0xbb, 0x6b, 0xb5, 0xa3, 0xcc, 0x8e, 0x87, 0x19, 0x4f, 0x92, 0xed, 0x81, 0x2b, 0x5c, - 0xb8, 0x23, 0xce, 0xfc, 0x31, 0x1c, 0xf9, 0x13, 0x50, 0xf8, 0x47, 0x90, 0x3f, 0x7b, 0x1e, 0xc9, - 0x6e, 0x9a, 0x2a, 0x70, 0xf3, 0xf7, 0x7e, 0xfd, 0x3e, 0xdb, 0x30, 0xc8, 0xf2, 0xf8, 0x84, 0x2b, + 0x0d, 0x3d, 0x44, 0xa5, 0xbd, 0xf0, 0xaa, 0x14, 0x25, 0x1b, 0x60, 0x10, 0x09, 0xe0, 0x49, 0x7a, + 0xeb, 0xc1, 0xdd, 0xb5, 0xda, 0x51, 0x66, 0xc7, 0xc3, 0x8c, 0x27, 0xc9, 0xf6, 0xc0, 0x15, 0x2e, + 0xdc, 0x11, 0x67, 0xfe, 0x18, 0x8e, 0xfc, 0x09, 0x28, 0xfc, 0x23, 0xc8, 0x9f, 0x3d, 0x8f, 0x64, + 0x37, 0x4d, 0x15, 0x7a, 0xf3, 0xf7, 0x7e, 0xfd, 0x3e, 0xdb, 0x30, 0xc8, 0xf2, 0xf8, 0x84, 0x2b, 0xb1, 0x95, 0xe5, 0x52, 0x49, 0xd2, 0x8b, 0x53, 0x25, 0xf2, 0x94, 0x27, 0x74, 0x19, 0xfa, 0x61, - 0x3a, 0x11, 0x67, 0xfb, 0x42, 0x71, 0xfa, 0x87, 0x03, 0xb7, 0x9e, 0xc4, 0x22, 0x99, 0x7c, 0x97, - 0xa9, 0x58, 0xa6, 0x05, 0xf9, 0x00, 0xfa, 0xbb, 0x7c, 0xfc, 0x52, 0x1c, 0xce, 0x32, 0x11, 0x78, - 0x43, 0x67, 0xb3, 0xcf, 0x1a, 0x46, 0x2d, 0x8d, 0xe2, 0x57, 0x22, 0xf0, 0x87, 0xce, 0xe6, 0x80, - 0x35, 0x0c, 0x32, 0x84, 0xe5, 0xc3, 0x78, 0x2a, 0x7e, 0x28, 0x79, 0xaa, 0xca, 0x69, 0xd0, 0x41, - 0xeb, 0x36, 0x8b, 0x10, 0xf0, 0xd1, 0x71, 0x0f, 0x45, 0x78, 0x26, 0xab, 0xe0, 0xed, 0xc7, 0x69, - 0xd0, 0x1f, 0x3a, 0x9b, 0x1e, 0xd3, 0x47, 0xe4, 0xf0, 0xb3, 0x00, 0x2c, 0x87, 0x9f, 0x51, 0x0a, - 0x2b, 0xe1, 0x34, 0x93, 0xb9, 0x62, 0xa2, 0xc8, 0x64, 0x5a, 0xa0, 0xd5, 0x5e, 0x9e, 0x07, 0x0e, - 0x3a, 0xd2, 0x47, 0xfa, 0x13, 0xac, 0xee, 0x24, 0x72, 0x7c, 0x3c, 0xe2, 0x8a, 0x33, 0xf1, 0x63, - 0x29, 0x0a, 0x45, 0xde, 0x81, 0x0e, 0xd6, 0x6a, 0xf5, 0x0c, 0xa1, 0xb9, 0x4f, 0x72, 0x3e, 0x15, - 0x81, 0x6b, 0xb8, 0x48, 0x68, 0x2e, 0xda, 0x63, 0xd5, 0x3e, 0x33, 0x84, 0xe6, 0x46, 0x49, 0x3c, - 0x36, 0xd5, 0xfa, 0xcc, 0x10, 0xba, 0x8e, 0xa7, 0xb1, 0x38, 0xb5, 0x25, 0xe2, 0x99, 0x86, 0xb0, - 0xd6, 0x8a, 0x6f, 0xd3, 0x5c, 0x87, 0x2e, 0x93, 0xa7, 0xe1, 0xa8, 0x08, 0x9c, 0xa1, 0xb7, 0xe9, - 0x33, 0x4b, 0x61, 0x23, 0x65, 0x52, 0x4e, 0x53, 0x2d, 0x72, 0x51, 0xd4, 0x30, 0xe8, 0x1d, 0xe8, - 0x60, 0x57, 0x75, 0x95, 0x8d, 0xad, 0x3e, 0xd2, 0x9f, 0x1d, 0xe8, 0xef, 0xf3, 0x33, 0x4c, 0xa3, - 0x20, 0x8f, 0xa1, 0x17, 0x29, 0x9e, 0x4e, 0x78, 0x3e, 0x41, 0xa5, 0xe5, 0x87, 0x1f, 0x6d, 0x55, - 0x83, 0xde, 0xaa, 0xd5, 0xb6, 0x2a, 0x9d, 0xbd, 0x54, 0xe5, 0x33, 0x56, 0x9b, 0xbc, 0xff, 0x05, - 0x0c, 0x2e, 0x88, 0x74, 0xbc, 0x63, 0x31, 0xab, 0xba, 0x7a, 0x2c, 0x66, 0xba, 0xfe, 0x13, 0x9e, - 0x94, 0xa6, 0x57, 0x3e, 0x33, 0xc4, 0xe7, 0xee, 0xa7, 0x0e, 0xdd, 0x06, 0xb2, 0x9b, 0x0b, 0xae, - 0x04, 0x06, 0xd9, 0x17, 0x45, 0xc1, 0x5f, 0x88, 0xab, 0x3b, 0x6e, 0xba, 0xe8, 0xb6, 0xba, 0x48, - 0xef, 0x03, 0x19, 0x89, 0x44, 0x28, 0x61, 0xf1, 0xf8, 0x1a, 0x0f, 0x34, 0xaa, 0xa2, 0x5d, 0xaf, - 0x4b, 0xee, 0x81, 0xaf, 0xc1, 0x8d, 0xc1, 0x96, 0x1f, 0xde, 0x6e, 0x3a, 0x52, 0xe3, 0x9e, 0xa1, - 0x02, 0x4d, 0x2a, 0xa7, 0x88, 0x80, 0x6b, 0x4b, 0x58, 0x00, 0x9a, 0xfb, 0x36, 0x94, 0x87, 0xa1, - 0xd6, 0x9b, 0x50, 0xed, 0xa5, 0xb2, 0xd1, 0xb6, 0xab, 0x72, 0x6f, 0x1a, 0x8d, 0x3e, 0x83, 0x0e, - 0xfa, 0xd5, 0xf8, 0x3b, 0xd0, 0x52, 0x63, 0x83, 0xe7, 0x3a, 0x15, 0xf7, 0xfa, 0x54, 0xb4, 0x7b, - 0x8d, 0xd9, 0x22, 0xf0, 0x86, 0x9e, 0x76, 0x8f, 0x04, 0x7d, 0x04, 0xdd, 0x68, 0xfc, 0x52, 0x4c, - 0x39, 0xf9, 0x18, 0x96, 0x30, 0x0f, 0x51, 0x58, 0x58, 0xbd, 0x7d, 0xa9, 0x89, 0xac, 0x92, 0xd3, - 0x91, 0xcd, 0x7f, 0x61, 0x4e, 0xf7, 0xa0, 0x8b, 0x99, 0x17, 0x81, 0x7f, 0xd9, 0x0d, 0x66, 0xc5, - 0xac, 0x98, 0xee, 0x81, 0x77, 0xc4, 0x42, 0xbd, 0x2e, 0x98, 0x41, 0xe5, 0xc5, 0x52, 0xda, 0xf7, - 0xd7, 0xb2, 0x50, 0xb6, 0x1b, 0x78, 0xd6, 0xbc, 0xef, 0x65, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x67, - 0xfa, 0x0c, 0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x7d, 0xb8, 0xe1, 0x88, 0x7c, - 0x88, 0xee, 0x6d, 0x6b, 0x06, 0x4d, 0x12, 0x47, 0x2c, 0x64, 0x18, 0xf8, 0x2e, 0x0c, 0xc2, 0x62, - 0x57, 0xca, 0x7c, 0x12, 0xa7, 0x5c, 0xc9, 0x1c, 0xbd, 0xf6, 0xd8, 0x45, 0x26, 0xdd, 0x86, 0x55, - 0xed, 0x3e, 0x52, 0x5c, 0xd5, 0xf3, 0x5b, 0x87, 0xae, 0xe6, 0xd5, 0xe1, 0x2c, 0x85, 0x90, 0xd7, - 0x7a, 0xd5, 0x04, 0x91, 0xa0, 0xdf, 0x1a, 0x0f, 0x7b, 0x27, 0x22, 0x55, 0x2d, 0x04, 0x20, 0x8d, - 0x0e, 0x06, 0xcc, 0x10, 0x84, 0x9a, 0x52, 0x6c, 0xce, 0x2b, 0x4d, 0xce, 0x9a, 0xcb, 0x50, 0x46, - 0x7f, 0x75, 0x00, 0xaa, 0x84, 0xca, 0xa2, 0x36, 0x71, 0xae, 0x36, 0x21, 0x9f, 0xb4, 0xae, 0x8f, - 0xf9, 0x05, 0xa9, 0x45, 0xac, 0x75, 0xc9, 0x6c, 0x56, 0xb0, 0xb0, 0x28, 0x5f, 0x6d, 0xf4, 0x0d, - 0xdf, 0x8e, 0x89, 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0xdb, 0x8c, 0xf4, 0x35, 0x67, - 0x18, 0x75, 0x7f, 0x1a, 0xc6, 0xe2, 0x16, 0x91, 0xbb, 0xd0, 0xd1, 0x99, 0x1a, 0x6c, 0xce, 0x97, - 0x61, 0x84, 0xf4, 0x29, 0xf4, 0x76, 0xa2, 0xf0, 0xab, 0x5c, 0x96, 0xd9, 0x42, 0xe4, 0x55, 0x2f, - 0x8d, 0x3b, 0xff, 0xd2, 0x78, 0x73, 0x2f, 0x8d, 0xdf, 0xbc, 0x34, 0x11, 0xac, 0x99, 0x2b, 0x41, - 0xaf, 0xc4, 0x4d, 0x6e, 0x84, 0xea, 0x69, 0xf0, 0x5a, 0x4f, 0x43, 0x04, 0x6b, 0x66, 0xf3, 0xff, - 0x4f, 0xa7, 0xbf, 0xbb, 0xb0, 0xc6, 0x44, 0x11, 0xbf, 0x12, 0x61, 0x5a, 0xa8, 0xbc, 0x1c, 0xeb, - 0x05, 0xd7, 0xf6, 0xdf, 0xc8, 0xe7, 0xb6, 0xdb, 0x1e, 0x33, 0xc4, 0x9b, 0x80, 0x89, 0x3c, 0x80, - 0xe5, 0xcb, 0x0b, 0x30, 0xaf, 0xda, 0x56, 0x21, 0x0f, 0x60, 0x29, 0x92, 0x65, 0x3e, 0xae, 0xd7, - 0xbb, 0x75, 0xe9, 0x98, 0xcc, 0x8c, 0x98, 0x55, 0x6a, 0x2d, 0x28, 0x75, 0x5e, 0x0f, 0x25, 0xf2, - 0xf8, 0x12, 0x94, 0x82, 0x2e, 0x1a, 0xbc, 0xd7, 0x18, 0x5c, 0x10, 0xb3, 0x8b, 0xda, 0xf4, 0x17, - 0x07, 0x6e, 0xb5, 0x53, 0x78, 0xa3, 0xdd, 0xa8, 0x27, 0xe2, 0x2e, 0x9c, 0x88, 0xb7, 0x68, 0x22, - 0x7e, 0x33, 0x91, 0xe6, 0x95, 0xeb, 0xb4, 0x5f, 0xb9, 0x63, 0xb8, 0x33, 0x37, 0xa6, 0x5d, 0x39, - 0xcd, 0x34, 0x1e, 0xfe, 0xc3, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0x76, 0x50, 0x7d, 0x66, 0x08, 0xfa, - 0x19, 0xbc, 0x1b, 0x09, 0xd5, 0x1a, 0x52, 0x85, 0xb6, 0x21, 0x78, 0x07, 0xe2, 0xf4, 0x8a, 0xf2, - 0xb5, 0x88, 0x7e, 0x09, 0xc1, 0x51, 0x36, 0xe1, 0x4a, 0xdc, 0xc8, 0x7a, 0x07, 0x7a, 0x87, 0x32, - 0x93, 0x89, 0x7c, 0x31, 0xbb, 0x66, 0xeb, 0x03, 0x58, 0x32, 0x57, 0xa4, 0xf9, 0xf8, 0xf4, 0x59, - 0x45, 0xd2, 0xdb, 0x1a, 0xd0, 0x63, 0x9e, 0x8c, 0xcb, 0x44, 0xa7, 0xa1, 0x7f, 0x40, 0xc5, 0xce, - 0xea, 0x9f, 0xe7, 0x1b, 0xce, 0x5f, 0xe7, 0x1b, 0xce, 0xdf, 0xe7, 0x1b, 0xce, 0x6f, 0xff, 0x6c, - 0xbc, 0xf5, 0xbc, 0x8b, 0x3f, 0xda, 0x47, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xab, 0x64, 0x15, - 0x6e, 0xe2, 0x0a, 0x00, 0x00, + 0x3a, 0x11, 0x67, 0xfb, 0x42, 0x71, 0xfa, 0xa7, 0x03, 0xb7, 0xbe, 0x8a, 0x45, 0x32, 0xf9, 0x3e, + 0x53, 0xb1, 0x4c, 0x0b, 0xf2, 0x01, 0xf4, 0x77, 0xf9, 0xf8, 0x85, 0x38, 0x9c, 0x65, 0x22, 0xf0, + 0x86, 0xce, 0x66, 0x9f, 0x35, 0x8c, 0x5a, 0x1a, 0xc5, 0x2f, 0x45, 0xe0, 0x0f, 0x9d, 0xcd, 0x01, + 0x6b, 0x18, 0x64, 0x08, 0xcb, 0x87, 0xf1, 0x54, 0xfc, 0x58, 0xf2, 0x54, 0x95, 0xd3, 0xa0, 0x83, + 0xd6, 0x6d, 0x16, 0x21, 0xe0, 0xa3, 0xe3, 0x1e, 0x8a, 0xf0, 0x4c, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, + 0xa0, 0x3f, 0x74, 0x36, 0x3d, 0xa6, 0x8f, 0xc8, 0xe1, 0x67, 0x01, 0x58, 0x0e, 0x3f, 0xa3, 0x14, + 0x56, 0xc2, 0x69, 0x26, 0x73, 0xc5, 0x44, 0x91, 0xc9, 0xb4, 0x40, 0xab, 0xbd, 0x3c, 0x0f, 0x1c, + 0x74, 0xa4, 0x8f, 0xf4, 0x67, 0x58, 0xdd, 0x49, 0xe4, 0xf8, 0x78, 0xc4, 0x15, 0x67, 0xe2, 0xa7, + 0x52, 0x14, 0x8a, 0xbc, 0x03, 0x1d, 0xac, 0xd5, 0xea, 0x19, 0x42, 0x73, 0xb1, 0xe6, 0xc0, 0x35, + 0x5c, 0x24, 0x34, 0x17, 0xed, 0xb1, 0x6a, 0x9f, 0x19, 0x42, 0x73, 0xa3, 0x24, 0x1e, 0x9b, 0x6a, + 0x7d, 0x66, 0x08, 0x5d, 0xc7, 0x93, 0x58, 0x9c, 0xda, 0x12, 0xf1, 0x4c, 0x43, 0x58, 0x6b, 0xc5, + 0xb7, 0x69, 0xae, 0x43, 0x97, 0xc9, 0xd3, 0x70, 0x54, 0x04, 0xce, 0xd0, 0xdb, 0xf4, 0x99, 0xa5, + 0xb0, 0x91, 0x32, 0x29, 0xa7, 0xa9, 0x16, 0xb9, 0x28, 0x6a, 0x18, 0xf4, 0x0e, 0x74, 0xb0, 0xab, + 0xba, 0xca, 0xc6, 0x56, 0x1f, 0xe9, 0x2f, 0x0e, 0xf4, 0xf7, 0xf9, 0x19, 0xa6, 0x51, 0x90, 0xc7, + 0xd0, 0x8b, 0x14, 0x4f, 0x27, 0x3c, 0x9f, 0xa0, 0xd2, 0xf2, 0xc3, 0x8f, 0xb6, 0xaa, 0x41, 0x6f, + 0xd5, 0x6a, 0x5b, 0x95, 0xce, 0x5e, 0xaa, 0xf2, 0x19, 0xab, 0x4d, 0xde, 0xff, 0x02, 0x06, 0x17, + 0x44, 0x3a, 0xde, 0xb1, 0x98, 0x55, 0x5d, 0x3d, 0x16, 0x33, 0x5d, 0xff, 0x09, 0x4f, 0x4a, 0x81, + 0xbd, 0xf2, 0x99, 0x21, 0x3e, 0x77, 0x3f, 0x75, 0xe8, 0x36, 0x90, 0xdd, 0x5c, 0x70, 0x25, 0x30, + 0xc8, 0xbe, 0x28, 0x0a, 0xfe, 0x5c, 0x5c, 0xdd, 0x71, 0xd3, 0x45, 0xb7, 0xd5, 0x45, 0x7a, 0x1f, + 0xc8, 0x48, 0x24, 0x42, 0x09, 0x8b, 0xc7, 0x57, 0x78, 0xa0, 0x51, 0x15, 0xed, 0x7a, 0x5d, 0x72, + 0x0f, 0x7c, 0x0d, 0x6e, 0x0c, 0xb6, 0xfc, 0xf0, 0x76, 0xd3, 0x91, 0x1a, 0xf7, 0x0c, 0x15, 0x68, + 0x52, 0x39, 0x45, 0x04, 0x5c, 0x5b, 0xc2, 0x02, 0xd0, 0xdc, 0xb7, 0xa1, 0x3c, 0x0c, 0xb5, 0xde, + 0x84, 0x6a, 0x2f, 0x95, 0x8d, 0xb6, 0x5d, 0x95, 0x7b, 0xd3, 0x68, 0xf4, 0xa9, 0xe5, 0x6a, 0xfc, + 0x1d, 0xf0, 0xa9, 0xb0, 0x36, 0x78, 0xae, 0x53, 0x71, 0xaf, 0x4f, 0x45, 0xbb, 0xd7, 0x98, 0x2d, + 0x02, 0x6f, 0xe8, 0x69, 0xf7, 0x48, 0xd0, 0x47, 0xd0, 0x8d, 0xc6, 0x2f, 0xc4, 0x94, 0x93, 0x8f, + 0x61, 0x09, 0xf3, 0x10, 0x85, 0x85, 0xd5, 0xdb, 0x97, 0x9a, 0xc8, 0x2a, 0x39, 0x1d, 0xd9, 0xfc, + 0x17, 0xe6, 0x74, 0x0f, 0xba, 0x18, 0xbd, 0x08, 0xfc, 0xcb, 0x6e, 0x90, 0xcf, 0xac, 0x98, 0xee, + 0x81, 0x77, 0xc4, 0x42, 0xbd, 0x2e, 0x98, 0x41, 0xe5, 0xc5, 0x52, 0xda, 0xf7, 0x37, 0xb2, 0x50, + 0xb6, 0x1b, 0x78, 0xd6, 0xbc, 0x1f, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x67, 0xfa, 0x14, 0xfc, + 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x7d, 0xb8, 0xe1, 0x88, 0x7c, 0x88, 0xee, 0x6d, + 0x6b, 0x06, 0x4d, 0x12, 0x47, 0x2c, 0x64, 0x18, 0xf8, 0x2e, 0x0c, 0xc2, 0x62, 0x57, 0xca, 0x7c, + 0x12, 0xa7, 0x5c, 0xc9, 0x1c, 0xbd, 0xf6, 0xd8, 0x45, 0x26, 0xdd, 0x86, 0x55, 0xed, 0x3e, 0x52, + 0x5c, 0xd5, 0x80, 0x5f, 0x87, 0xae, 0xe6, 0xd5, 0xe1, 0x2c, 0x85, 0x90, 0xd7, 0x7a, 0xd5, 0x04, + 0x91, 0xa0, 0xdf, 0x19, 0x0f, 0x7b, 0x27, 0x22, 0x55, 0x2d, 0x04, 0x20, 0x8d, 0x0e, 0x06, 0xcc, + 0x10, 0x84, 0x9a, 0x52, 0x6c, 0xce, 0x2b, 0x4d, 0xce, 0x9a, 0xcb, 0x50, 0x46, 0x7f, 0x73, 0x00, + 0xaa, 0x84, 0xca, 0xa2, 0x36, 0x71, 0xae, 0x36, 0x21, 0x9f, 0xb4, 0xae, 0x8f, 0xf9, 0x05, 0xa9, + 0x45, 0xac, 0x75, 0xc9, 0x6c, 0x56, 0xb0, 0xb0, 0x28, 0x5f, 0x6d, 0xf4, 0x0d, 0xdf, 0x8e, 0x89, + 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0xdb, 0x8c, 0xf4, 0x35, 0x67, 0x18, 0x75, 0x7f, + 0x1a, 0xc6, 0xe2, 0x16, 0x91, 0xbb, 0xd0, 0xd1, 0x99, 0x1a, 0x6c, 0xce, 0x97, 0x61, 0x84, 0xf4, + 0x09, 0xf4, 0x76, 0xa2, 0xf0, 0xeb, 0x5c, 0x96, 0xd9, 0x42, 0xe4, 0x55, 0x2f, 0x8d, 0x3b, 0xff, + 0xd2, 0x78, 0x73, 0x2f, 0x8d, 0xdf, 0xbc, 0x34, 0x11, 0xac, 0x99, 0x2b, 0x41, 0xaf, 0xc4, 0x4d, + 0x6e, 0x84, 0xea, 0x69, 0xf0, 0x5a, 0x4f, 0x43, 0x04, 0x6b, 0x66, 0xf3, 0xdf, 0xa4, 0xd3, 0x3f, + 0x5c, 0x58, 0x63, 0xa2, 0x88, 0x5f, 0x8a, 0x30, 0x2d, 0x54, 0x5e, 0x8e, 0xf5, 0x82, 0x6b, 0xfb, + 0x6f, 0xe5, 0x33, 0xdb, 0x6d, 0x8f, 0x19, 0xe2, 0x75, 0xc0, 0x44, 0x1e, 0xc0, 0xf2, 0xe5, 0x05, + 0x98, 0x57, 0x6d, 0xab, 0x90, 0x07, 0xb0, 0x14, 0xc9, 0x32, 0xd7, 0x48, 0x32, 0xeb, 0xdd, 0xba, + 0x74, 0x4c, 0x66, 0x46, 0xcc, 0x2a, 0xb5, 0x16, 0x94, 0x3a, 0xaf, 0x86, 0x12, 0x79, 0x7c, 0x09, + 0x4a, 0x41, 0x17, 0x0d, 0xde, 0x6b, 0x0c, 0x2e, 0x88, 0xd9, 0x45, 0x6d, 0xfa, 0xab, 0x03, 0xb7, + 0xda, 0x29, 0xbc, 0xd6, 0x6e, 0xd4, 0x13, 0x71, 0x17, 0x4e, 0xc4, 0x5b, 0x34, 0x11, 0xbf, 0x99, + 0x48, 0xf3, 0xca, 0x75, 0xda, 0xaf, 0xdc, 0x31, 0xdc, 0x99, 0x1b, 0xd3, 0xae, 0x9c, 0x66, 0x1a, + 0x0f, 0xff, 0x63, 0x5c, 0xfa, 0xd6, 0xc8, 0x73, 0x3b, 0xa8, 0x3e, 0x33, 0x04, 0xfd, 0x0c, 0xde, + 0x8d, 0x84, 0x6a, 0x0d, 0xa9, 0x42, 0xdb, 0x10, 0xbc, 0x03, 0x71, 0x7a, 0x45, 0xf9, 0x5a, 0x44, + 0xbf, 0x84, 0xe0, 0x28, 0x9b, 0x70, 0x25, 0x6e, 0x64, 0xbd, 0x03, 0xbd, 0x43, 0x99, 0xc9, 0x44, + 0x3e, 0x9f, 0x5d, 0xb3, 0xf5, 0x01, 0x2c, 0x99, 0x2b, 0xd2, 0x7c, 0x7c, 0xfa, 0xac, 0x22, 0xe9, + 0x6d, 0x0d, 0xe8, 0x31, 0x4f, 0xc6, 0x65, 0xa2, 0xd3, 0xd0, 0x3f, 0xa0, 0x62, 0x67, 0xf5, 0xaf, + 0xf3, 0x0d, 0xe7, 0xef, 0xf3, 0x0d, 0xe7, 0x9f, 0xf3, 0x0d, 0xe7, 0xf7, 0x7f, 0x37, 0xde, 0x7a, + 0xd6, 0xc5, 0x1f, 0xed, 0xa3, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x97, 0xf0, 0x12, 0xfd, 0xe2, + 0x0a, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 30d41205b..9cab31828 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -20,7 +20,7 @@ message ImportResponse { message BlockDataRequest { string Index = 1; - string Frame = 2; + string Field = 2; string View = 5; uint64 Slice = 4; uint64 Block = 3; @@ -53,15 +53,15 @@ message CreateIndexMessage { IndexMeta Meta = 2; } -message CreateFrameMessage { +message CreateFieldMessage { string Index = 1; - string Frame = 2; + string Field = 2; FieldOptions Meta = 3; } -message DeleteFrameMessage { +message DeleteFieldMessage { string Index = 1; - string Frame = 2; + string Field = 2; } message Field { @@ -76,7 +76,7 @@ message Schema { message Index { string Name = 1; - repeated Field Frames = 4; + repeated Field Fields = 4; } message URI { @@ -122,13 +122,13 @@ message BSIGroup { message CreateViewMessage { string Index = 1; - string Frame = 2; + string Field = 2; string View = 3; } message DeleteViewMessage { string Index = 1; - string Frame = 2; + string Field = 2; string View = 3; } @@ -144,7 +144,7 @@ message ResizeInstruction { message ResizeSource { Node Node = 1; string Index = 2; - string Frame = 3; + string Field = 3; string View = 4; uint64 Slice = 5; } diff --git a/internal/public.pb.go b/internal/public.pb.go index 2fc4b3827..76266d2db 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -411,7 +411,7 @@ func (m *QueryResult) GetChanged() bool { type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` @@ -432,9 +432,9 @@ func (m *ImportRequest) GetIndex() string { return "" } -func (m *ImportRequest) GetFrame() string { +func (m *ImportRequest) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -483,7 +483,7 @@ func (m *ImportRequest) GetTimestamps() []int64 { type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` @@ -502,9 +502,9 @@ func (m *ImportValueRequest) GetIndex() string { return "" } -func (m *ImportValueRequest) GetFrame() string { +func (m *ImportValueRequest) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -1054,11 +1054,11 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if m.Slice != 0 { dAtA[i] = 0x18 @@ -1171,11 +1171,11 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if m.Slice != 0 { dAtA[i] = 0x18 @@ -1474,7 +1474,7 @@ func (m *ImportRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPublic(uint64(l)) } @@ -1524,7 +1524,7 @@ func (m *ImportValueRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPublic(uint64(l)) } @@ -3072,7 +3072,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3097,7 +3097,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 0 { @@ -3443,7 +3443,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3468,7 +3468,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 0 { @@ -3771,49 +3771,49 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 701 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd4, 0x3c, - 0x14, 0xfd, 0x3c, 0xc9, 0xfc, 0xdd, 0xe9, 0xcc, 0x57, 0x59, 0xdf, 0x57, 0x22, 0x84, 0x86, 0x28, - 0x42, 0x28, 0xab, 0xa9, 0x34, 0xec, 0x41, 0xf4, 0x4f, 0x1a, 0x55, 0x54, 0x70, 0x5b, 0x8a, 0x58, - 0xa6, 0xad, 0x55, 0x22, 0x65, 0xe2, 0x90, 0x38, 0x9a, 0xce, 0x73, 0xb0, 0xe1, 0x11, 0x58, 0xf0, - 0x10, 0x2c, 0xbb, 0xe4, 0x11, 0xa0, 0xbc, 0x08, 0xf2, 0x75, 0x3c, 0x49, 0xa7, 0x52, 0xc5, 0x82, - 0x9d, 0xcf, 0x39, 0xf6, 0xb5, 0x8f, 0x7d, 0x6e, 0x02, 0x1b, 0x59, 0x79, 0x96, 0xc4, 0xe7, 0x93, - 0x2c, 0x97, 0x4a, 0xf2, 0x5e, 0x9c, 0x2a, 0x91, 0xa7, 0x51, 0x12, 0xbc, 0x07, 0x07, 0xe5, 0x82, - 0x7b, 0xd0, 0xdd, 0x95, 0x49, 0x39, 0x4f, 0x0b, 0x8f, 0xf9, 0x4e, 0xe8, 0xa2, 0x85, 0xfc, 0x09, - 0xb4, 0x5f, 0x2a, 0x95, 0x17, 0x5e, 0xcb, 0x77, 0xc2, 0xc1, 0x74, 0x34, 0xb1, 0x4b, 0x27, 0x9a, - 0x46, 0x23, 0x72, 0x0e, 0xee, 0xa1, 0x58, 0x16, 0x9e, 0xe3, 0x3b, 0x61, 0x1f, 0x69, 0x1c, 0x3c, - 0x07, 0xf7, 0x75, 0x14, 0xe7, 0x7c, 0x04, 0xad, 0xd9, 0x9e, 0xc7, 0x7c, 0x16, 0xba, 0xd8, 0x9a, - 0xed, 0xf1, 0xff, 0xa0, 0xbd, 0x2b, 0xcb, 0x54, 0x79, 0x2d, 0xa2, 0x0c, 0xe0, 0x9b, 0xe0, 0x1c, - 0x8a, 0xa5, 0xe7, 0xf8, 0x2c, 0xec, 0xa3, 0x1e, 0x06, 0x53, 0xe8, 0x9d, 0x46, 0xc9, 0x4a, 0x3d, - 0x8d, 0x12, 0x2a, 0xe2, 0xa0, 0x1e, 0xde, 0xae, 0xe2, 0x54, 0x55, 0x82, 0xb7, 0xe0, 0xec, 0xc4, - 0x4a, 0x8b, 0x28, 0x17, 0xab, 0x5d, 0x0d, 0xe0, 0x0f, 0xa1, 0x67, 0x5c, 0xcd, 0xf6, 0xaa, 0xbd, - 0x57, 0x98, 0x3f, 0x82, 0xfe, 0x49, 0x3c, 0x17, 0x85, 0x8a, 0xe6, 0x19, 0x1d, 0xc2, 0xc1, 0x9a, - 0x08, 0xde, 0xc1, 0xd0, 0xcc, 0xd4, 0x6e, 0x8f, 0x85, 0xba, 0xe3, 0xe9, 0xcf, 0x6e, 0xe9, 0xae, - 0xc7, 0x2f, 0x0c, 0x5c, 0xad, 0x59, 0x89, 0xad, 0x24, 0x7d, 0xa5, 0x27, 0xcb, 0x4c, 0x54, 0x27, - 0xa5, 0x31, 0xf7, 0x61, 0x70, 0xac, 0xf2, 0x38, 0xbd, 0x3c, 0x8d, 0x92, 0x52, 0x54, 0x85, 0x9a, - 0x94, 0xf6, 0x38, 0x4b, 0x95, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xda, 0xe3, 0x8e, 0x94, 0x89, 0x11, - 0xdb, 0x3e, 0x0b, 0x7b, 0x58, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xc8, 0xa8, 0x5a, 0xdb, 0xf1, 0x59, - 0xc8, 0xb0, 0xc1, 0x04, 0xdb, 0xd0, 0xd5, 0x27, 0x7d, 0x15, 0x65, 0xb5, 0x5b, 0x76, 0x8f, 0xdb, - 0xe0, 0x9a, 0xc1, 0xc6, 0x9b, 0x52, 0xe4, 0x4b, 0x14, 0x1f, 0x4b, 0x51, 0xd0, 0xab, 0x10, 0xae, - 0x5c, 0x1a, 0xc0, 0xb7, 0xa0, 0x73, 0x9c, 0xc4, 0xe7, 0xc2, 0xdc, 0x9d, 0x8b, 0x15, 0xd2, 0x5e, - 0xeb, 0x3b, 0x2f, 0xc8, 0x6b, 0x0f, 0x9b, 0x94, 0x5e, 0x89, 0x62, 0x2e, 0x95, 0x35, 0x53, 0x21, - 0x1e, 0xc2, 0xbf, 0xfb, 0x57, 0xe7, 0x49, 0x79, 0x21, 0x50, 0x2e, 0xcc, 0xea, 0x0e, 0x4d, 0x58, - 0xa7, 0xf9, 0x53, 0x18, 0x55, 0x94, 0x4d, 0x7f, 0x97, 0x26, 0xae, 0xb1, 0xc1, 0x27, 0x06, 0xc3, - 0xca, 0x4a, 0x91, 0xc9, 0xb4, 0x10, 0xfa, 0xbd, 0xf6, 0xf3, 0xdc, 0xbe, 0xd7, 0x7e, 0x9e, 0xf3, - 0x6d, 0xe8, 0xa2, 0x28, 0xca, 0x44, 0xd9, 0x10, 0xfc, 0x5f, 0x5f, 0x8b, 0x5d, 0x5b, 0x26, 0x0a, - 0xed, 0x2c, 0xfe, 0x02, 0x46, 0xb7, 0x42, 0x65, 0xba, 0x67, 0x30, 0x7d, 0x50, 0xaf, 0xbb, 0xa5, - 0xe3, 0xda, 0xf4, 0xe0, 0x1b, 0x83, 0x41, 0xa3, 0x32, 0x7f, 0x4c, 0xbd, 0x4c, 0x67, 0x1a, 0x4c, - 0x87, 0x75, 0x15, 0x94, 0x0b, 0xa4, 0x2e, 0xdf, 0x00, 0x76, 0x54, 0xe5, 0x89, 0x1d, 0xe9, 0x57, - 0xd4, 0xfd, 0x69, 0xb7, 0x6d, 0xbc, 0xa2, 0xa6, 0xd1, 0x88, 0xf4, 0x65, 0xf8, 0x10, 0xa5, 0x97, - 0xe2, 0x82, 0xf2, 0xd4, 0x43, 0x0b, 0xf9, 0xa4, 0xee, 0x4f, 0x7a, 0x80, 0xc1, 0x94, 0xd7, 0x25, - 0xac, 0x82, 0x75, 0x0f, 0xdb, 0x40, 0xeb, 0xb7, 0x18, 0x9a, 0x40, 0x07, 0x3f, 0x19, 0x0c, 0x67, - 0xf3, 0x4c, 0xe6, 0xaa, 0x11, 0x92, 0x59, 0x7a, 0x21, 0xae, 0x6c, 0x48, 0x08, 0x68, 0xf6, 0x20, - 0x8f, 0xe6, 0xa6, 0x1b, 0xfa, 0x68, 0x80, 0x66, 0x29, 0x2c, 0x14, 0x0e, 0x17, 0x0d, 0xa0, 0x58, - 0xe8, 0x7e, 0x2f, 0x3c, 0xd7, 0x04, 0xca, 0x20, 0x1d, 0x7f, 0xdb, 0xee, 0x85, 0xd7, 0x26, 0xa9, - 0x26, 0x74, 0xfc, 0x57, 0xfd, 0xae, 0xf3, 0xe2, 0x84, 0x0e, 0x36, 0x18, 0x7d, 0x0f, 0x28, 0x17, - 0xf4, 0x91, 0xeb, 0xd2, 0x47, 0xce, 0x42, 0xbd, 0xd2, 0x94, 0x21, 0xb1, 0x47, 0x62, 0x83, 0x09, - 0xbe, 0x32, 0xe0, 0xc6, 0x23, 0x35, 0xd2, 0xdf, 0x33, 0x7a, 0xbf, 0xa1, 0x2d, 0xe8, 0xd0, 0x7e, - 0xd6, 0x4c, 0x85, 0xd6, 0x8e, 0xdb, 0x5d, 0x3f, 0xee, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbf, - 0x19, 0xb3, 0x1f, 0x37, 0x63, 0xf6, 0xf9, 0xd7, 0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x34, 0x9e, 0xfd, - 0x0e, 0x00, 0x00, 0xff, 0xff, 0xc8, 0x5d, 0x77, 0x8c, 0x44, 0x06, 0x00, 0x00, + // 699 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40, + 0x14, 0x65, 0x62, 0xe7, 0x75, 0xd3, 0x84, 0x6a, 0x04, 0xc5, 0x42, 0x28, 0x58, 0x16, 0x42, 0x5e, + 0xa5, 0x52, 0xd8, 0x83, 0xe8, 0x4b, 0x8a, 0x2a, 0x2a, 0xb8, 0x2d, 0x45, 0x2c, 0xdd, 0x66, 0x54, + 0x2c, 0x39, 0x9e, 0x60, 0x8f, 0x95, 0xe6, 0x3b, 0xd8, 0xf0, 0x09, 0x2c, 0xf8, 0x08, 0x96, 0x5d, + 0xf2, 0x09, 0x50, 0x7e, 0x04, 0xcd, 0x1d, 0x4f, 0xec, 0xa6, 0x52, 0xc5, 0x82, 0xdd, 0x9c, 0x73, + 0x66, 0xee, 0xcc, 0x99, 0x39, 0xd7, 0x86, 0x8d, 0x79, 0x71, 0x96, 0xc4, 0xe7, 0xa3, 0x79, 0x26, + 0x95, 0xe4, 0x9d, 0x38, 0x55, 0x22, 0x4b, 0xa3, 0x24, 0xf8, 0x08, 0x0e, 0xca, 0x05, 0xf7, 0xa0, + 0xbd, 0x2b, 0x93, 0x62, 0x96, 0xe6, 0x1e, 0xf3, 0x9d, 0xd0, 0x45, 0x0b, 0xf9, 0x33, 0x68, 0xbe, + 0x56, 0x2a, 0xcb, 0xbd, 0x86, 0xef, 0x84, 0xbd, 0xf1, 0x60, 0x64, 0x97, 0x8e, 0x34, 0x8d, 0x46, + 0xe4, 0x1c, 0xdc, 0x43, 0xb1, 0xcc, 0x3d, 0xc7, 0x77, 0xc2, 0x2e, 0xd2, 0x38, 0x78, 0x09, 0xee, + 0xdb, 0x28, 0xce, 0xf8, 0x00, 0x1a, 0x93, 0x3d, 0x8f, 0xf9, 0x2c, 0x74, 0xb1, 0x31, 0xd9, 0xe3, + 0x0f, 0xa0, 0xb9, 0x2b, 0x8b, 0x54, 0x79, 0x0d, 0xa2, 0x0c, 0xe0, 0x9b, 0xe0, 0x1c, 0x8a, 0xa5, + 0xe7, 0xf8, 0x2c, 0xec, 0xa2, 0x1e, 0x06, 0x63, 0xe8, 0x9c, 0x46, 0xc9, 0x4a, 0x3d, 0x8d, 0x12, + 0x2a, 0xe2, 0xa0, 0x1e, 0xde, 0xac, 0xe2, 0x94, 0x55, 0x82, 0xf7, 0xe0, 0xec, 0xc4, 0x4a, 0x8b, + 0x28, 0x17, 0xab, 0x5d, 0x0d, 0xe0, 0x8f, 0xa1, 0x63, 0x5c, 0x4d, 0xf6, 0xca, 0xbd, 0x57, 0x98, + 0x3f, 0x81, 0xee, 0x49, 0x3c, 0x13, 0xb9, 0x8a, 0x66, 0x73, 0x3a, 0x84, 0x83, 0x15, 0x11, 0x7c, + 0x80, 0xbe, 0x99, 0xa9, 0xdd, 0x1e, 0x0b, 0x75, 0xcb, 0xd3, 0xbf, 0xdd, 0xd2, 0x6d, 0x8f, 0xdf, + 0x18, 0xb8, 0x5a, 0xb3, 0x12, 0x5b, 0x49, 0xfa, 0x4a, 0x4f, 0x96, 0x73, 0x51, 0x9e, 0x94, 0xc6, + 0xdc, 0x87, 0xde, 0xb1, 0xca, 0xe2, 0xf4, 0xe2, 0x34, 0x4a, 0x0a, 0x51, 0x16, 0xaa, 0x53, 0xda, + 0xe3, 0x24, 0x55, 0x46, 0x76, 0xc9, 0xc6, 0x0a, 0x6b, 0x8f, 0x3b, 0x52, 0x26, 0x46, 0x6c, 0xfa, + 0x2c, 0xec, 0x60, 0x45, 0xf0, 0x21, 0xc0, 0x41, 0x22, 0xa3, 0x72, 0x6d, 0xcb, 0x67, 0x21, 0xc3, + 0x1a, 0x13, 0x6c, 0x43, 0x5b, 0x9f, 0xf4, 0x4d, 0x34, 0xaf, 0xdc, 0xb2, 0x3b, 0xdc, 0x06, 0x57, + 0x0c, 0x36, 0xde, 0x15, 0x22, 0x5b, 0xa2, 0xf8, 0x5c, 0x88, 0x9c, 0x5e, 0x85, 0x70, 0xe9, 0xd2, + 0x00, 0xbe, 0x05, 0xad, 0xe3, 0x24, 0x3e, 0x17, 0xe6, 0xee, 0x5c, 0x2c, 0x91, 0xf6, 0x5a, 0xdd, + 0x79, 0x4e, 0x5e, 0x3b, 0x58, 0xa7, 0xf4, 0x4a, 0x14, 0x33, 0xa9, 0xac, 0x99, 0x12, 0xf1, 0x10, + 0xee, 0xef, 0x5f, 0x9e, 0x27, 0xc5, 0x54, 0xa0, 0x5c, 0x98, 0xd5, 0x2d, 0x9a, 0xb0, 0x4e, 0xf3, + 0xe7, 0x30, 0x28, 0x29, 0x9b, 0xfe, 0x36, 0x4d, 0x5c, 0x63, 0x83, 0x2f, 0x0c, 0xfa, 0xa5, 0x95, + 0x7c, 0x2e, 0xd3, 0x5c, 0xe8, 0xf7, 0xda, 0xcf, 0x32, 0xfb, 0x5e, 0xfb, 0x59, 0xc6, 0xb7, 0xa1, + 0x8d, 0x22, 0x2f, 0x12, 0x65, 0x43, 0xf0, 0xb0, 0xba, 0x16, 0xbb, 0xb6, 0x48, 0x14, 0xda, 0x59, + 0xfc, 0x15, 0x0c, 0x6e, 0x84, 0xca, 0x74, 0x4f, 0x6f, 0xfc, 0xa8, 0x5a, 0x77, 0x43, 0xc7, 0xb5, + 0xe9, 0xc1, 0x0f, 0x06, 0xbd, 0x5a, 0x65, 0xfe, 0x94, 0x7a, 0x99, 0xce, 0xd4, 0x1b, 0xf7, 0xab, + 0x2a, 0x28, 0x17, 0x48, 0x5d, 0xbe, 0x01, 0xec, 0xa8, 0xcc, 0x13, 0x3b, 0xd2, 0xaf, 0xa8, 0xfb, + 0xd3, 0x6e, 0x5b, 0x7b, 0x45, 0x4d, 0xa3, 0x11, 0xe9, 0xcb, 0xf0, 0x29, 0x4a, 0x2f, 0xc4, 0x94, + 0xf2, 0xd4, 0x41, 0x0b, 0xf9, 0xa8, 0xea, 0x4f, 0x7a, 0x80, 0xde, 0x98, 0x57, 0x25, 0xac, 0x82, + 0x55, 0x0f, 0xdb, 0x40, 0xeb, 0xb7, 0xe8, 0x9b, 0x40, 0x07, 0xbf, 0x19, 0xf4, 0x27, 0xb3, 0xb9, + 0xcc, 0x54, 0x2d, 0x24, 0x93, 0x74, 0x2a, 0x2e, 0x6d, 0x48, 0x08, 0x68, 0xf6, 0x20, 0x16, 0xc9, + 0x94, 0x4e, 0xdf, 0x45, 0x03, 0x34, 0x4b, 0x61, 0xa1, 0x70, 0xb8, 0x68, 0x00, 0xc5, 0x42, 0xf7, + 0x7b, 0xee, 0xb9, 0x26, 0x50, 0x06, 0xe9, 0xf8, 0xdb, 0x76, 0xcf, 0xbd, 0x26, 0x49, 0x15, 0xa1, + 0xe3, 0xbf, 0xea, 0x77, 0x9d, 0x17, 0x27, 0x74, 0xb0, 0xc6, 0xe8, 0x7b, 0x40, 0xb9, 0xa0, 0x8f, + 0x5c, 0x9b, 0x3e, 0x72, 0x16, 0xea, 0x95, 0xa6, 0x0c, 0x89, 0x1d, 0x12, 0x6b, 0x4c, 0xf0, 0x9d, + 0x01, 0x37, 0x1e, 0xa9, 0x91, 0xfe, 0x9f, 0xd1, 0xbb, 0x0d, 0x6d, 0x41, 0x8b, 0xf6, 0xb3, 0x66, + 0x4a, 0xb4, 0x76, 0xdc, 0xf6, 0xfa, 0x71, 0x77, 0x36, 0xaf, 0xae, 0x87, 0xec, 0xe7, 0xf5, 0x90, + 0xfd, 0xba, 0x1e, 0xb2, 0xaf, 0x7f, 0x86, 0xf7, 0xce, 0x5a, 0xf4, 0xd3, 0x78, 0xf1, 0x37, 0x00, + 0x00, 0xff, 0xff, 0x71, 0x71, 0xa2, 0x0c, 0x44, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index b37eea98c..cd34b88ff 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -70,7 +70,7 @@ message QueryResult { message ImportRequest { string Index = 1; - string Frame = 2; + string Field = 2; uint64 Slice = 3; repeated uint64 RowIDs = 4; repeated uint64 ColumnIDs = 5; @@ -81,7 +81,7 @@ message ImportRequest { message ImportValueRequest { string Index = 1; - string Frame = 2; + string Field = 2; uint64 Slice = 3; repeated uint64 ColumnIDs = 5; repeated string ColumnKeys = 7; diff --git a/server.go b/server.go index 541ff12c6..471319dd7 100644 --- a/server.go +++ b/server.go @@ -455,34 +455,34 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err := s.Holder.DeleteIndex(obj.Index); err != nil { return err } - case *internal.CreateFrameMessage: + case *internal.CreateFieldMessage: idx := s.Holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } opt := decodeFieldOptions(obj.Meta) - _, err := idx.CreateField(obj.Frame, *opt) + _, err := idx.CreateField(obj.Field, *opt) if err != nil { return err } - case *internal.DeleteFrameMessage: + case *internal.DeleteFieldMessage: idx := s.Holder.Index(obj.Index) - if err := idx.DeleteField(obj.Frame); err != nil { + if err := idx.DeleteField(obj.Field); err != nil { return err } case *internal.CreateViewMessage: - f := s.Holder.Field(obj.Index, obj.Frame) + f := s.Holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local Frame not found: %s", obj.Frame) + return fmt.Errorf("Local Field not found: %s", obj.Field) } _, _, err := f.createViewIfNotExistsBase(obj.View) if err != nil { return err } case *internal.DeleteViewMessage: - f := s.Holder.Field(obj.Index, obj.Frame) + f := s.Holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local Frame not found: %s", obj.Frame) + return fmt.Errorf("Local Field not found: %s", obj.Field) } err := f.DeleteView(obj.View) if err != nil { From 4c44c5f33a06ffa2f7b03be764f623a14d806fd4 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 23:33:00 -0500 Subject: [PATCH 46/78] fix Frame to Field in tests --- server/cluster_test.go | 10 +++++----- server/server_test.go | 12 ++++++------ server_test.go | 2 +- stats_test.go | 8 ++++---- test/cluster.go | 8 ++++---- utils_test.go | 8 ++++---- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/server/cluster_test.go b/server/cluster_test.go index f445ce415..9327befc0 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -54,7 +54,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Create indexes and frames on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -209,7 +209,7 @@ func TestClusterResize_AddNode(t *testing.T) { // Create indexes and frames on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -253,7 +253,7 @@ func TestClusterResize_AddNode(t *testing.T) { // Create indexes and frames on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -305,7 +305,7 @@ func TestClusterResize_AddNode(t *testing.T) { // Create indexes and frames on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -458,7 +458,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // Create indexes and frames on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } diff --git a/server/server_test.go b/server/server_test.go index 0fa45619f..495812cdf 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -54,7 +54,7 @@ func TestMain_Set_Quick(t *testing.T) { if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) } - if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FieldOptions{}); err != nil && err != pilosa.ErrFieldExists { + if err := client.CreateField(context.Background(), "i", cmd.Frame, pilosa.FieldOptions{}); err != nil && err != pilosa.ErrFieldExists { t.Fatal(err) } if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { @@ -123,11 +123,11 @@ func TestMain_SetRowAttrs(t *testing.T) { client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "z", pilosa.FieldOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "z", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "neg", pilosa.FieldOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "neg", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -200,7 +200,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -286,7 +286,7 @@ func TestMain_RecalculateHashes(t *testing.T) { if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal("create index:", err) } - if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FieldOptions{CacheType: "ranked"}); err != nil { + if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{CacheType: "ranked"}); err != nil { t.Fatal("create frame:", err) } diff --git a/server_test.go b/server_test.go index 9b5409330..5ea5653cc 100644 --- a/server_test.go +++ b/server_test.go @@ -19,7 +19,7 @@ func TestMonitorAntiEntropy(t *testing.T) { if err != nil { t.Fatalf("creating index: %v", err) } - err = client.CreateFrame(context.Background(), "balh", "fralh", pilosa.FieldOptions{}) + err = client.CreateField(context.Background(), "balh", "fralh", pilosa.FieldOptions{}) if err != nil { t.Fatalf("creating frame: %v", err) } diff --git a/stats_test.go b/stats_test.go index 7f71cfabb..806baa738 100644 --- a/stats_test.go +++ b/stats_test.go @@ -272,8 +272,8 @@ func TestStatsCount_CreateFrame(t *testing.T) { called := false s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != "createFrame" { - t.Errorf("Expected createFrame, Results %s", name) + if name != "createField" { + t.Errorf("Expected createField, Results %s", name) } if index[0] != "index:i" { t.Errorf("Expected index:i, Results %s", index) @@ -303,8 +303,8 @@ func TestStatsCount_DeleteFrame(t *testing.T) { } s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != "deleteFrame" { - t.Errorf("Expected deleteFrame, Results %s", name) + if name != "deleteField" { + t.Errorf("Expected deleteField, Results %s", name) } if index[0] != "index:i" { t.Errorf("Expected index:i, Results %s", index) diff --git a/test/cluster.go b/test/cluster.go index 26ef44d30..ee87d51a2 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -367,7 +367,7 @@ func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) if err := func() error { // figure out which node it was meant for, then call the operation on that cluster - // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI) instrNode := pilosa.DecodeNode(instr.Node) destCluster := t.clusterByID(instrNode.ID) @@ -380,11 +380,11 @@ func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) srcNode := pilosa.DecodeNode(src.Node) srcCluster := t.clusterByID(srcNode.ID) - srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) - destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + srcFragment := srcCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice) + destFragment := destCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice) if destFragment == nil { // Create fragment on destination if it doesn't exist. - f := destCluster.Holder.Field(src.Index, src.Frame) + f := destCluster.Holder.Field(src.Index, src.Field) v := f.View(src.View) var err error destFragment, err = v.CreateFragmentIfNotExists(src.Slice) diff --git a/utils_test.go b/utils_test.go index 34b4f2967..15053cd2b 100644 --- a/utils_test.go +++ b/utils_test.go @@ -356,7 +356,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi if err := func() error { // figure out which node it was meant for, then call the operation on that cluster - // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI) instrNode := DecodeNode(instr.Node) destCluster := t.clusterByID(instrNode.ID) @@ -369,11 +369,11 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi srcNode := DecodeNode(src.Node) srcCluster := t.clusterByID(srcNode.ID) - srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) - destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + srcFragment := srcCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice) + destFragment := destCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice) if destFragment == nil { // Create fragment on destination if it doesn't exist. - f := destCluster.Holder.Field(src.Index, src.Frame) + f := destCluster.Holder.Field(src.Index, src.Field) v := f.View(src.View) var err error destFragment, err = v.CreateFragmentIfNotExists(src.Slice) From 4e0aff0ca2a5c3a70bf0b5efc1c6b1e1e5e51b73 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 23:43:01 -0500 Subject: [PATCH 47/78] finish frame to field in api.go --- api.go | 126 ++++++++++++++++++++++++++--------------------------- handler.go | 6 +-- 2 files changed, 66 insertions(+), 66 deletions(-) diff --git a/api.go b/api.go index ae010e4f1..56eaece56 100644 --- a/api.go +++ b/api.go @@ -217,9 +217,9 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { return nil } -// CreateFrame makes the named frame in the named index with the given options. -func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FieldOptions) (*Field, error) { - if err := api.validate(apiCreateFrame); err != nil { +// CreateField makes the named field in the named index with the given options. +func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, options FieldOptions) (*Field, error) { + if err := api.validate(apiCreateField); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -229,17 +229,17 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str return nil, ErrIndexNotFound } - // Create frame. - frame, err := index.CreateField(frameName, options) + // Create field. + field, err := index.CreateField(fieldName, options) if err != nil { - return nil, errors.Wrap(err, "creating frame") + return nil, errors.Wrap(err, "creating field") } - // Send the create frame message to all nodes. + // Send the create field message to all nodes. err = api.Broadcaster.SendSync( &internal.CreateFieldMessage{ Index: indexName, - Field: frameName, + Field: fieldName, Meta: options.Encode(), }) if err != nil { @@ -247,14 +247,14 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str return nil, errors.Wrap(err, "sending CreateField message") } api.Holder.Stats.CountWithCustomTags("createField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) - return frame, nil + return field, nil } -// DeleteFrame removes the named frame from the named index. If the index is not -// found, an error is returned. If the frame is not found, it is ignored and no +// DeleteField removes the named field from the named index. If the index is not +// found, an error is returned. If the field is not found, it is ignored and no // action is taken. -func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { - if err := api.validate(apiDeleteFrame); err != nil { +func (api *API) DeleteField(ctx context.Context, indexName string, fieldName string) error { + if err := api.validate(apiDeleteField); err != nil { return errors.Wrap(err, "validating api method") } @@ -264,16 +264,16 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str return ErrIndexNotFound } - // Delete frame from the index. - if err := index.DeleteField(frameName); err != nil { - return errors.Wrap(err, "deleting frame") + // Delete field from the index. + if err := index.DeleteField(fieldName); err != nil { + return errors.Wrap(err, "deleting field") } - // Send the delete frame message to all nodes. + // Send the delete field message to all nodes. err := api.Broadcaster.SendSync( &internal.DeleteFieldMessage{ Index: indexName, - Field: frameName, + Field: fieldName, }) if err != nil { api.Logger.Printf("problem sending DeleteField message: %s", err) @@ -283,9 +283,9 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str return nil } -// ExportCSV encodes the fragment designated by the index,frame,slice as +// ExportCSV encodes the fragment designated by the index,field,slice as // CSV of the form , -func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, slice uint64, w io.Writer) error { +func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, slice uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { return errors.Wrap(err, "validating api method") } @@ -297,7 +297,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin } // Find the fragment. - f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) if f == nil { return ErrFragmentNotFound } @@ -333,13 +333,13 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) // MarshalFragment returns an object which can write the specified fragment's data // to an io.Writer. The serialized data can be read back into a fragment with // the UnmarshalFragment API call. -func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64) (io.WriterTo, error) { +func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64) (io.WriterTo, error) { if err := api.validate(apiMarshalFragment); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) if f == nil { return nil, ErrFragmentNotFound } @@ -349,13 +349,13 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // UnmarshalFragment creates a new fragment (if necessary) and reads data from a // Reader which was previously written by MarshalFragment to populate the // fragment's data. -func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64, reader io.ReadCloser) error { +func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64, reader io.ReadCloser) error { if err := api.validate(apiUnmarshalFragment); err != nil { return errors.Wrap(err, "validating api method") } - // Retrieve frame. - f := api.Holder.Field(indexName, frameName) + // Retrieve field. + f := api.Holder.Field(indexName, fieldName) if f == nil { return ErrFieldNotFound } @@ -366,7 +366,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa return errors.Wrap(err, "creating view") } - // Retrieve fragment from frame. + // Retrieve fragment from field. frag, err := view.CreateFragmentIfNotExists(slice) if err != nil { return errors.Wrap(err, "creating fragment") @@ -415,13 +415,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. -func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, slice uint64) ([]FragmentBlock, error) { +func (api *API) FragmentBlocks(ctx context.Context, indexName string, fieldName string, slice uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) if f == nil { return nil, ErrFragmentNotFound } @@ -482,20 +482,20 @@ func (api *API) LocalID() string { return api.Cluster.Node.ID } -// Schema returns information about each index in Pilosa including which frames +// Schema returns information about each index in Pilosa including which fields // and views they contain. func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.Holder.Schema() } -// Views returns the views in the given frame. -func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { +// Views returns the views in the given field. +func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*View, error) { if err := api.validate(apiViews); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve views. - f := api.Holder.Field(indexName, frameName) + f := api.Holder.Field(indexName, fieldName) if f == nil { return nil, ErrFieldNotFound } @@ -506,13 +506,13 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( } // DeleteView removes the given view. -func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { +func (api *API) DeleteView(ctx context.Context, indexName string, fieldName string, viewName string) error { if err := api.validate(apiDeleteView); err != nil { return errors.Wrap(err, "validating api method") } - // Retrieve frame. - f := api.Holder.Field(indexName, frameName) + // Retrieve field. + f := api.Holder.Field(indexName, fieldName) if f == nil { return ErrFieldNotFound } @@ -529,7 +529,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri err := api.Broadcaster.SendSync( &internal.DeleteViewMessage{ Index: indexName, - Field: frameName, + Field: fieldName, View: viewName, }) if err != nil { @@ -574,13 +574,13 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At return attrs, nil } -func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { - if err := api.validate(apiFrameAttrDiff); err != nil { +func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + if err := api.validate(apiFieldAttrDiff); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve index from holder. - f := api.Holder.Field(indexName, frameName) + f := api.Holder.Field(indexName, fieldName) if f == nil { return nil, ErrFieldNotFound } @@ -608,15 +608,15 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s return attrs, nil } -// Import bulk imports data into a particular index,frame,slice. +// Import bulk imports data into a particular index,field,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { if err := api.validate(apiImport); err != nil { return errors.Wrap(err, "validating api method") } - _, frame, err := api.indexFrame(req.Index, req.Field, req.Slice) + _, field, err := api.indexField(req.Index, req.Field, req.Slice) if err != nil { - return errors.Wrap(err, "getting frame") + return errors.Wrap(err, "getting field") } // Convert timestamps to time.Time. @@ -630,9 +630,9 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { } // Import into fragment. - err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps) + err = field.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } @@ -643,15 +643,15 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest return errors.Wrap(err, "validating api method") } - _, frame, err := api.indexFrame(req.Index, req.Field, req.Slice) + _, field, err := api.indexField(req.Index, req.Field, req.Slice) if err != nil { - return errors.Wrap(err, "getting frame") + return errors.Wrap(err, "getting field") } // Import into fragment. - err = frame.ImportValue(req.ColumnIDs, req.Values) + err = field.ImportValue(req.ColumnIDs, req.Values) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } @@ -679,7 +679,7 @@ func (api *API) LongQueryTime() time.Duration { return api.Cluster.LongQueryTime } -func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Field, error) { +func (api *API) indexField(indexName string, fieldName string, slice uint64) (*Index, *Field, error) { // Validate that this handler owns the slice. if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) @@ -687,20 +687,20 @@ func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*I } // Find the Index. - api.Logger.Printf("importing: %v %v %v", indexName, frameName, slice) + api.Logger.Printf("importing: %v %v %v", indexName, fieldName, slice) index := api.Holder.Index(indexName) if index == nil { - api.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error()) + api.Logger.Printf("fragment error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrIndexNotFound.Error()) return nil, nil, ErrIndexNotFound } - // Retrieve frame. - frame := index.Field(frameName) - if frame == nil { - api.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFieldNotFound.Error()) + // Retrieve field. + field := index.Field(fieldName) + if field == nil { + api.Logger.Printf("field error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrFieldNotFound.Error()) return nil, nil, ErrFieldNotFound } - return index, frame, nil + return index, field, nil } // SetCoordinator makes a new Node the cluster coordinator. @@ -790,15 +790,15 @@ type apiMethod int // API validation constants. const ( apiClusterMessage apiMethod = iota - apiCreateFrame + apiCreateField apiCreateIndex - apiDeleteFrame + apiDeleteField apiDeleteIndex apiDeleteView apiExportCSV apiFragmentBlockData apiFragmentBlocks - apiFrameAttrDiff + apiFieldAttrDiff //apiHosts // not implemented apiImport apiImportValue @@ -833,15 +833,15 @@ var methodsResizing = map[apiMethod]struct{}{ } var methodsNormal = map[apiMethod]struct{}{ - apiCreateFrame: struct{}{}, + apiCreateField: struct{}{}, apiCreateIndex: struct{}{}, - apiDeleteFrame: struct{}{}, + apiDeleteField: struct{}{}, apiDeleteIndex: struct{}{}, apiDeleteView: struct{}{}, apiExportCSV: struct{}{}, apiFragmentBlockData: struct{}{}, apiFragmentBlocks: struct{}{}, - apiFrameAttrDiff: struct{}{}, + apiFieldAttrDiff: struct{}{}, apiImport: struct{}{}, apiImportValue: struct{}{}, apiIndex: struct{}{}, diff --git a/handler.go b/handler.go index a57a40204..dca880aba 100644 --- a/handler.go +++ b/handler.go @@ -517,7 +517,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - _, err = h.API.CreateFrame(r.Context(), indexName, frameName, req.Options) + _, err = h.API.CreateField(r.Context(), indexName, frameName, req.Options) if err != nil { switch errors.Cause(err) { case ErrIndexNotFound: @@ -585,7 +585,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - err := h.API.DeleteFrame(r.Context(), indexName, frameName) + err := h.API.DeleteField(r.Context(), indexName, frameName) if err != nil { if errors.Cause(err) == ErrIndexNotFound { if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { @@ -617,7 +617,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request return } - attrs, err := h.API.FrameAttrDiff(r.Context(), indexName, frameName, req.Blocks) + attrs, err := h.API.FieldAttrDiff(r.Context(), indexName, frameName, req.Blocks) if err != nil { switch errors.Cause(err) { case ErrFragmentNotFound: From ba5d687ee0b44cce3f2ee30d218356c517116f36 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 23:46:17 -0500 Subject: [PATCH 48/78] finish frame to field in frame.go --- frame.go | 86 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/frame.go b/frame.go index 60e41e265..0e1f79d66 100644 --- a/frame.go +++ b/frame.go @@ -29,13 +29,13 @@ import ( "github.com/pkg/errors" ) -// Default frame settings. +// Default field settings. const ( DefaultFieldType = FieldTypeSet DefaultCacheType = CacheTypeRanked - // Default ranked frame cache + // Default ranked field cache DefaultCacheSize = 50000 ) @@ -80,7 +80,7 @@ func OptFieldFieldOptions(o FieldOptions) FieldOption { } } -// NewField returns a new instance of frame. +// NewField returns a new instance of field. func NewField(path, index, name string, opts ...FieldOption) (*Field, error) { err := ValidateName(name) if err != nil { @@ -118,19 +118,19 @@ func NewField(path, index, name string, opts ...FieldOption) (*Field, error) { return f, nil } -// Name returns the name the frame was initialized with. +// Name returns the name the field was initialized with. func (f *Field) Name() string { return f.name } -// Index returns the index name the frame was initialized with. +// Index returns the index name the field was initialized with. func (f *Field) Index() string { return f.index } -// Path returns the path the frame was initialized with. +// Path returns the path the field was initialized with. func (f *Field) Path() string { return f.path } // RowAttrStore returns the attribute storage. func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore } -// MaxSlice returns the max slice in the frame. +// MaxSlice returns the max slice in the field. func (f *Field) MaxSlice() uint64 { f.mu.RLock() defer f.mu.RUnlock() @@ -144,14 +144,14 @@ func (f *Field) MaxSlice() uint64 { return max } -// Type returns the frame type. +// Type returns the field type. func (f *Field) Type() string { f.mu.RLock() defer f.mu.RUnlock() return f.options.Type } -// CacheType returns the caching mode for the frame. +// CacheType returns the caching mode for the field. func (f *Field) CacheType() string { f.mu.RLock() defer f.mu.RUnlock() @@ -178,7 +178,7 @@ func (f *Field) SetCacheSize(v uint32) error { return nil } -// CacheSize returns the ranked frame cache size. +// CacheSize returns the ranked field cache size. func (f *Field) CacheSize() uint32 { f.mu.RLock() v := f.options.CacheSize @@ -186,26 +186,26 @@ func (f *Field) CacheSize() uint32 { return v } -// Options returns all options for this frame. +// Options returns all options for this field. func (f *Field) Options() FieldOptions { f.mu.RLock() defer f.mu.RUnlock() return f.options } -// Open opens and initializes the frame. +// Open opens and initializes the field. func (f *Field) Open() error { if err := func() error { - // Ensure the frame's path exists. + // Ensure the field's path exists. if err := os.MkdirAll(f.path, 0777); err != nil { - return errors.Wrap(err, "creating frame dir") + return errors.Wrap(err, "creating field dir") } if err := f.loadMeta(); err != nil { return errors.Wrap(err, "loading meta") } - // Apply the frame options loaded from meta. + // Apply the field options loaded from meta. if err := f.applyOptions(f.options); err != nil { return errors.Wrap(err, "applying options") } @@ -227,7 +227,7 @@ func (f *Field) Open() error { return nil } -// openViews opens and initializes the views inside the frame. +// openViews opens and initializes the views inside the field. func (f *Field) openViews() error { file, err := os.Open(filepath.Join(f.path, "views")) if os.IsNotExist(err) { @@ -259,7 +259,7 @@ func (f *Field) openViews() error { return nil } -// loadMeta reads meta data for the frame, if any. +// loadMeta reads meta data for the field, if any. func (f *Field) loadMeta() error { var pb internal.FieldOptions @@ -286,7 +286,7 @@ func (f *Field) loadMeta() error { return nil } -// saveMeta writes meta data for the frame. +// saveMeta writes meta data for the field. func (f *Field) saveMeta() error { // Marshal metadata. fo := f.options @@ -303,7 +303,7 @@ func (f *Field) saveMeta() error { return nil } -// applyOptions configures the frame based on opt. +// applyOptions configures the field based on opt. func (f *Field) applyOptions(opt FieldOptions) error { switch opt.Type { case FieldTypeSet, "": @@ -351,13 +351,13 @@ func (f *Field) applyOptions(opt FieldOptions) error { return errors.Wrap(err, "setting time quantum") } default: - return errors.New("invalid frame type") + return errors.New("invalid field type") } return nil } -// Close closes the frame and its views. +// Close closes the field and its views. func (f *Field) Close() error { f.mu.Lock() defer f.mu.Unlock() @@ -390,7 +390,7 @@ func (f *Field) bsiGroup(name string) *bsiGroup { return nil } -// hasBSIGroup returns true if a bsiGroup exists on the frame. +// hasBSIGroup returns true if a bsiGroup exists on the field. func (f *Field) hasBSIGroup(name string) bool { for _, bsig := range f.bsiGroups { if bsig.Name == name { @@ -400,7 +400,7 @@ func (f *Field) hasBSIGroup(name string) bool { return false } -// createBSIGroup creates a new bsiGroup on the frame. +// createBSIGroup creates a new bsiGroup on the field. func (f *Field) createBSIGroup(bsig *bsiGroup) error { f.mu.Lock() defer f.mu.Unlock() @@ -469,14 +469,14 @@ func (f *Field) deleteBSIGroup(name string) error { return ErrBSIGroupNotFound } -// TimeQuantum returns the time quantum for the frame. +// TimeQuantum returns the time quantum for the field. func (f *Field) TimeQuantum() TimeQuantum { f.mu.Lock() defer f.mu.Unlock() return f.options.TimeQuantum } -// SetTimeQuantum sets the time quantum for the frame. +// SetTimeQuantum sets the time quantum for the field. func (f *Field) SetTimeQuantum(q TimeQuantum) error { f.mu.Lock() defer f.mu.Unlock() @@ -486,7 +486,7 @@ func (f *Field) SetTimeQuantum(q TimeQuantum) error { return ErrInvalidTimeQuantum } - // Update value on frame. + // Update value on field. f.options.TimeQuantum = q // Persist meta data to disk. @@ -497,12 +497,12 @@ func (f *Field) SetTimeQuantum(q TimeQuantum) error { return nil } -// ViewPath returns the path to a view in the frame. +// ViewPath returns the path to a view in the field. func (f *Field) ViewPath(name string) string { return filepath.Join(f.path, "views", name) } -// View returns a view in the frame by name. +// View returns a view in the field by name. func (f *Field) View(name string) *View { f.mu.RLock() defer f.mu.RUnlock() @@ -511,7 +511,7 @@ func (f *Field) View(name string) *View { func (f *Field) view(name string) *View { return f.views[name] } -// Views returns a list of all views in the frame. +// Views returns a list of all views in the field. func (f *Field) Views() []*View { f.mu.RLock() defer f.mu.RUnlock() @@ -523,7 +523,7 @@ func (f *Field) Views() []*View { return other } -// viewNames returns a list of all views (as a string) in the frame. +// viewNames returns a list of all views (as a string) in the field. func (f *Field) viewNames() []string { f.mu.Lock() defer f.mu.Unlock() @@ -535,7 +535,7 @@ func (f *Field) viewNames() []string { return other } -// RecalculateCaches recalculates caches on every view in the frame. +// RecalculateCaches recalculates caches on every view in the field. func (f *Field) RecalculateCaches() { for _, view := range f.Views() { view.RecalculateCaches() @@ -598,7 +598,7 @@ func (f *Field) newView(path, name string) *View { return view } -// DeleteView removes the view from the frame. +// DeleteView removes the view from the field. func (f *Field) DeleteView(name string) error { view := f.views[name] if view == nil { @@ -620,7 +620,7 @@ func (f *Field) DeleteView(name string) error { return nil } -// SetBit sets a bit on a view within the frame. +// SetBit sets a bit on a view within the field. func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. if !IsValidView(name) { @@ -662,7 +662,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed return changed, nil } -// ClearBit clears a bit within the frame. +// ClearBit clears a bit within the field. func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. if !IsValidView(name) { @@ -704,7 +704,7 @@ func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (change return changed, nil } -// Value reads a frame value for a column. +// Value reads a field value for a column. func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { bsig := f.bsiGroup(f.name) if bsig == nil { @@ -726,7 +726,7 @@ func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { return int64(v) + bsig.Min, true, nil } -// SetValue sets a frame value for a column. +// SetValue sets a field value for a column. func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) { // Fetch bsiGroup and validate value. bsig := f.bsiGroup(f.name) @@ -750,7 +750,7 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) return view.setValue(columnID, bsig.BitDepth(), baseValue) } -// Sum returns the sum and count for a frame. +// Sum returns the sum and count for a field. // An optional filtering row can be provided. func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) { bsig := f.bsiGroup(name) @@ -770,7 +770,7 @@ func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) { return int64(vsum) + (int64(vcount) * bsig.Min), int64(vcount), nil } -// Min returns the min for a frame. +// Min returns the min for a field. // An optional filtering row can be provided. func (f *Field) Min(filter *Row, name string) (min, count int64, err error) { bsig := f.bsiGroup(name) @@ -790,7 +790,7 @@ func (f *Field) Min(filter *Row, name string) (min, count int64, err error) { return int64(vmin) + bsig.Min, int64(vcount), nil } -// Max returns the max for a frame. +// Max returns the max for a field. // An optional filtering row can be provided. func (f *Field) Max(filter *Row, name string) (max, count int64, err error) { bsig := f.bsiGroup(name) @@ -861,7 +861,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // Determine quantum if timestamps are set. q := f.TimeQuantum() if hasTime(timestamps) && q == "" { - return errors.New("time quantum not set in either index or frame") + return errors.New("time quantum not set in field") } // Split import data by fragment. @@ -995,7 +995,7 @@ func (p fieldSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p fieldSlice) Len() int { return len(p) } func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } -// FieldInfo represents schema information for a frame. +// FieldInfo represents schema information for a field. type FieldInfo struct { Name string `json:"name"` Options FieldOptions `json:"options"` @@ -1032,7 +1032,7 @@ func (o *FieldOptions) Validate() error { return ErrInvalidTimeQuantum } default: - return errors.New("invalid frame type") + return errors.New("invalid field type") } return nil } @@ -1084,7 +1084,7 @@ func isValidBSIGroupType(v string) bool { } } -// bsiGroup represents a group of range-encoded rows on a frame. +// bsiGroup represents a group of range-encoded rows on a field. type bsiGroup struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` From 0f8bd62e3354ea8896c68faf9686e62497b51e00 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 5 Jun 2018 23:58:46 -0500 Subject: [PATCH 49/78] more Frame to Field in tests. move frame*.go files to field*.go --- client_test.go | 2 +- executor_test.go | 22 +++---- frame.go => field.go | 0 ...internal_test.go => field_internal_test.go | 0 frame_test.go => field_test.go | 66 +++++++++---------- fragment_test.go | 4 +- handler_test.go | 16 ++--- index_test.go | 10 +-- test/frame.go | 44 ++++++------- test/holder.go | 8 +-- test/index.go | 12 ++-- 11 files changed, 92 insertions(+), 92 deletions(-) rename frame.go => field.go (100%) rename frame_internal_test.go => field_internal_test.go (100%) rename frame_test.go => field_test.go (77%) diff --git a/client_test.go b/client_test.go index 6f1931e29..9091c8200 100644 --- a/client_test.go +++ b/client_test.go @@ -254,7 +254,7 @@ func TestClient_ImportValue(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frame, err := index.CreateFrameIfNotExists(fldName, fo) + frame, err := index.CreateFieldIfNotExists(fldName, fo) if err != nil { t.Fatal(err) } diff --git a/executor_test.go b/executor_test.go index af4278849..bd965c325 100644 --- a/executor_test.go +++ b/executor_test.go @@ -33,7 +33,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := index.CreateFrame("f", pilosa.FieldOptions{}) + f, err := index.CreateField("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } @@ -83,7 +83,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrame("f", pilosa.FieldOptions{}); err != nil { + if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -271,13 +271,13 @@ func TestExecutor_Execute_SetValue(t *testing.T) { // Create frames. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{ + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 0, Max: 50, }); err != nil { t.Fatal(err) - } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FieldOptions{}); err != nil { + } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -311,7 +311,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{ + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 0, Max: 100, @@ -349,9 +349,9 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Create frames. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{}); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FieldOptions{}); err != nil { + } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -737,7 +737,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) // Create frame. - if _, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{ + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeTime, TimeQuantum: pilosa.TimeQuantum("YMDH"), }); err != nil { @@ -1068,7 +1068,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Create frame. - if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FieldOptions{}); err != nil { + if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -1120,7 +1120,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Create frame. - if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FieldOptions{}); err != nil { + if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := f.SetTimeQuantum("Y"); err != nil { t.Fatal(err) @@ -1223,7 +1223,7 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - index.CreateFrame("f", pilosa.FieldOptions{}) + index.CreateField("f", pilosa.FieldOptions{}) targetAttrs := map[string]interface{}{ "foo": "bar", } diff --git a/frame.go b/field.go similarity index 100% rename from frame.go rename to field.go diff --git a/frame_internal_test.go b/field_internal_test.go similarity index 100% rename from frame_internal_test.go rename to field_internal_test.go diff --git a/frame_test.go b/field_test.go similarity index 77% rename from frame_test.go rename to field_test.go index 99dea0938..10e4d9a97 100644 --- a/frame_test.go +++ b/field_test.go @@ -22,9 +22,9 @@ import ( "github.com/pilosa/pilosa/test" ) -// Ensure frame can open and retrieve a view. -func TestFrame_CreateViewIfNotExists(t *testing.T) { - f := test.MustOpenFrame() +// Ensure field can open and retrieve a view. +func TestField_CreateViewIfNotExists(t *testing.T) { + f := test.MustOpenField() defer f.Close() // Create view. @@ -48,12 +48,12 @@ func TestFrame_CreateViewIfNotExists(t *testing.T) { } } -// Ensure frame can set its time quantum. -func TestFrame_SetTimeQuantum(t *testing.T) { +// Ensure field can set its time quantum. +func TestField_SetTimeQuantum(t *testing.T) { fo := pilosa.FieldOptions{ Type: "time", } - f := test.MustOpenFrame(pilosa.OptFieldFieldOptions(fo)) + f := test.MustOpenField(pilosa.OptFieldFieldOptions(fo)) defer f.Close() // Set & retrieve time quantum. @@ -63,7 +63,7 @@ func TestFrame_SetTimeQuantum(t *testing.T) { t.Fatalf("unexpected quantum: %s", q) } - // Reload frame and verify that it is persisted. + // Reload field and verify that it is persisted. if err := f.Reopen(); err != nil { t.Fatal(err) } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { @@ -71,13 +71,13 @@ func TestFrame_SetTimeQuantum(t *testing.T) { } } -// Ensure a frame can set & read a bsiGroup value. -func TestFrame_SetValue(t *testing.T) { +// Ensure a field can set & read a bsiGroup value. +func TestField_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateFrame("f", pilosa.FieldOptions{ + f, err := idx.CreateField("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 0, Max: 30, @@ -86,7 +86,7 @@ func TestFrame_SetValue(t *testing.T) { t.Fatal(err) } - // Set value on frame. + // Set value on field. if changed, err := f.SetValue(100, 21); err != nil { t.Fatal(err) } else if !changed { @@ -114,7 +114,7 @@ func TestFrame_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateFrame("f", pilosa.FieldOptions{ + f, err := idx.CreateField("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 0, Max: 30, @@ -151,7 +151,7 @@ func TestFrame_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateFrame("f", pilosa.FieldOptions{ + f, err := idx.CreateField("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeSet, }) if err != nil { @@ -168,7 +168,7 @@ func TestFrame_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateFrame("f", pilosa.FieldOptions{ + f, err := idx.CreateField("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 20, Max: 30, @@ -187,7 +187,7 @@ func TestFrame_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateFrame("f", pilosa.FieldOptions{ + f, err := idx.CreateField("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 20, Max: 30, @@ -203,27 +203,27 @@ func TestFrame_SetValue(t *testing.T) { }) } -func TestFrame_NameRestriction(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa-frame-") +func TestField_NameRestriction(t *testing.T) { + path, err := ioutil.TempDir("", "pilosa-field-") if err != nil { panic(err) } - frame, err := pilosa.NewField(path, "i", ".meta") - if frame != nil { - t.Fatalf("unexpected frame name %s", err) + field, err := pilosa.NewField(path, "i", ".meta") + if field != nil { + t.Fatalf("unexpected field name %s", err) } } -// Ensure that frame name validation is consistent. -func TestFrame_NameValidation(t *testing.T) { - validFrameNames := []string{ +// Ensure that field name validation is consistent. +func TestField_NameValidation(t *testing.T) { + validFieldNames := []string{ "foo", "hyphen-ated", "under_score", "abc123", "trailing_", } - invalidFrameNames := []string{ + invalidFieldNames := []string{ "", "123abc", "x.y", @@ -235,27 +235,27 @@ func TestFrame_NameValidation(t *testing.T) { "a12345678901234567890123456789012345678901234567890123456789012345", } - path, err := ioutil.TempDir("", "pilosa-frame-") + path, err := ioutil.TempDir("", "pilosa-field-") if err != nil { panic(err) } - for _, name := range validFrameNames { + for _, name := range validFieldNames { _, err := pilosa.NewField(path, "i", name) if err != nil { - t.Fatalf("unexpected frame name: %s %s", name, err) + t.Fatalf("unexpected field name: %s %s", name, err) } } - for _, name := range invalidFrameNames { + for _, name := range invalidFieldNames { _, err := pilosa.NewField(path, "i", name) if err == nil { - t.Fatalf("expected error on frame name: %s", name) + t.Fatalf("expected error on field name: %s", name) } } } -// Ensure frame can open and retrieve a view. -func TestFrame_DeleteView(t *testing.T) { - f := test.MustOpenFrame() +// Ensure field can open and retrieve a view. +func TestField_DeleteView(t *testing.T) { + f := test.MustOpenField() defer f.Close() viewName := pilosa.ViewStandard + "_v" @@ -274,7 +274,7 @@ func TestFrame_DeleteView(t *testing.T) { } if f.View(viewName) != nil { - t.Fatal("view still exists in frame") + t.Fatal("view still exists in field") } // Recreate view with same name, verify that the old view was not reused. diff --git a/fragment_test.go b/fragment_test.go index 71dff2b71..7f140e907 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -754,7 +754,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { defer index.Close() // Create frame. - frame, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) + frame, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) if err != nil { t.Fatal(err) } @@ -924,7 +924,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { defer index.Close() // Create frame. - frame, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) + frame, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) if err != nil { t.Fatal(err) } diff --git a/handler_test.go b/handler_test.go index 74b1702fa..bd6449862 100644 --- a/handler_test.go +++ b/handler_test.go @@ -83,17 +83,17 @@ func TestHandler_Schema(t *testing.T) { i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FieldOptions{}); err != nil { + if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -120,17 +120,17 @@ func TestHandler_Status(t *testing.T) { i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FieldOptions{}); err != nil { + if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -699,7 +699,7 @@ func TestHandler_DeleteFrame(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FieldOptions{}); err != nil { + if _, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -777,7 +777,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { // Set attributes on the index. idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("meta", pilosa.FieldOptions{}) + f, err := idx.CreateFieldIfNotExists("meta", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } diff --git a/index_test.go b/index_test.go index ada9be7f1..2272fd5bd 100644 --- a/index_test.go +++ b/index_test.go @@ -29,7 +29,7 @@ func TestIndex_CreateFrameIfNotExists(t *testing.T) { defer index.Close() // Create frame. - f, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{}) + f, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } else if f == nil { @@ -37,7 +37,7 @@ func TestIndex_CreateFrameIfNotExists(t *testing.T) { } // Retrieve existing frame. - other, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{}) + other, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } else if f.Field != other.Field { @@ -57,7 +57,7 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() // Create frame with explicit quantum. - f, err := index.CreateFrame("f", pilosa.FieldOptions{ + f, err := index.CreateField("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeTime, TimeQuantum: pilosa.TimeQuantum("YMDH"), }) @@ -76,7 +76,7 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() // Create frame with schema and verify it exists. - if f, err := index.CreateFrame("f", pilosa.FieldOptions{ + if f, err := index.CreateField("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 10, Max: 20, @@ -180,7 +180,7 @@ func TestIndex_DeleteFrame(t *testing.T) { defer index.Close() // Create frame. - if _, err := index.CreateFrameIfNotExists("f", pilosa.FieldOptions{}); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } diff --git a/test/frame.go b/test/frame.go index 929070368..75dc5800d 100644 --- a/test/frame.go +++ b/test/frame.go @@ -23,41 +23,41 @@ import ( "github.com/pilosa/pilosa" ) -// Frame represents a test wrapper for pilosa.Frame. -type Frame struct { +// Field represents a test wrapper for pilosa.Field. +type Field struct { *pilosa.Field } -// NewFrame returns a new instance of Frame d/0. -func NewFrame(opt ...pilosa.FieldOption) *Frame { - path, err := ioutil.TempDir("", "pilosa-frame-") +// NewField returns a new instance of Field d/0. +func NewField(opt ...pilosa.FieldOption) *Field { + path, err := ioutil.TempDir("", "pilosa-field-") if err != nil { panic(err) } - frame, err := pilosa.NewField(path, "i", "f", opt...) + field, err := pilosa.NewField(path, "i", "f", opt...) if err != nil { panic(err) } - return &Frame{Field: frame} + return &Field{Field: field} } -// MustOpenFrame returns a new, opened frame at a temporary path. Panic on error. -func MustOpenFrame(opt ...pilosa.FieldOption) *Frame { - f := NewFrame(opt...) +// MustOpenField returns a new, opened field at a temporary path. Panic on error. +func MustOpenField(opt ...pilosa.FieldOption) *Field { + f := NewField(opt...) if err := f.Open(); err != nil { panic(err) } return f } -// Close closes the frame and removes the underlying data. -func (f *Frame) Close() error { +// Close closes the field and removes the underlying data. +func (f *Field) Close() error { defer os.RemoveAll(f.Path()) return f.Field.Close() } // Reopen closes the index and reopens it. -func (f *Frame) Reopen() error { +func (f *Field) Reopen() error { var err error if err := f.Field.Close(); err != nil { return err @@ -75,8 +75,8 @@ func (f *Frame) Reopen() error { return nil } -// MustSetBit sets a bit on the frame. Panic on error. -func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) { +// MustSetBit sets a bit on the field. Panic on error. +func (f *Field) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) { changed, err := f.SetBit(view, rowID, columnID, t) if err != nil { panic(err) @@ -84,23 +84,23 @@ func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (c return changed } -// Ensure frame can set its cache -func TestFrame_SetCacheSize(t *testing.T) { - f := MustOpenFrame() +// Ensure field can set its cache +func TestField_SetCacheSize(t *testing.T) { + f := MustOpenField() defer f.Close() cacheSize := uint32(100) - // Set & retrieve frame cache size. + // Set & retrieve field cache size. if err := f.SetCacheSize(cacheSize); err != nil { t.Fatal(err) } else if q := f.CacheSize(); q != cacheSize { - t.Fatalf("unexpected frame cache size: %d", q) + t.Fatalf("unexpected field cache size: %d", q) } - // Reload frame and verify that it is persisted. + // Reload field and verify that it is persisted. if err := f.Reopen(); err != nil { t.Fatal(err) } else if q := f.CacheSize(); q != cacheSize { - t.Fatalf("unexpected frame cache size (reopen): %d", q) + t.Fatalf("unexpected field cache size (reopen): %d", q) } } diff --git a/test/holder.go b/test/holder.go index 1344bd4c9..c0e8f1233 100644 --- a/test/holder.go +++ b/test/holder.go @@ -81,8 +81,8 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption } // MustCreateFrameIfNotExists returns a given frame. Panic on error. -func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame { - f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FieldOptions{}) +func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Field { + f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(frame, pilosa.FieldOptions{}) if err != nil { panic(err) } @@ -92,7 +92,7 @@ func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame { // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists(frame, pilosa.FieldOptions{}) + f, err := idx.CreateFieldIfNotExists(frame, pilosa.FieldOptions{}) if err != nil { panic(err) } @@ -110,7 +110,7 @@ func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice // MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists(frame, pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) + f, err := idx.CreateFieldIfNotExists(frame, pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) if err != nil { panic(err) } diff --git a/test/index.go b/test/index.go index 93fa99cad..69c18682c 100644 --- a/test/index.go +++ b/test/index.go @@ -73,20 +73,20 @@ func (i *Index) Reopen() error { return nil } -// CreateFrame creates a frame with the given options. -func (i *Index) CreateFrame(name string, opt pilosa.FieldOptions) (*Frame, error) { +// CreateField creates a field with the given options. +func (i *Index) CreateField(name string, opt pilosa.FieldOptions) (*Field, error) { f, err := i.Index.CreateField(name, opt) if err != nil { return nil, err } - return &Frame{Field: f}, nil + return &Field{Field: f}, nil } -// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. -func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FieldOptions) (*Frame, error) { +// CreateFieldIfNotExists creates a field with the given options if it doesn't exist. +func (i *Index) CreateFieldIfNotExists(name string, opt pilosa.FieldOptions) (*Field, error) { f, err := i.Index.CreateFieldIfNotExists(name, opt) if err != nil { return nil, err } - return &Frame{Field: f}, nil + return &Field{Field: f}, nil } From 2a9b1e9e5b30a67211eae42b1be0dfd009285d81 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 6 Jun 2018 01:27:12 -0500 Subject: [PATCH 50/78] final Frame to Field rename --- apimethod_string.go | 2 +- client.go | 16 ++-- client_test.go | 14 +-- cluster_internal_test.go | 18 ++-- cluster_test.go | 10 +- cmd/export.go | 2 +- cmd/export_test.go | 4 +- cmd/import.go | 10 +- cmd/import_test.go | 4 +- ctl/export.go | 8 +- ctl/export_test.go | 4 +- ctl/import.go | 40 ++++---- ctl/import_test.go | 10 +- executor.go | 196 +++++++++++++++++++-------------------- executor_test.go | 174 +++++++++++++++++----------------- fragment.go | 4 +- fragment_test.go | 12 +-- handler.go | 84 ++++++++--------- handler_internal_test.go | 12 +-- handler_test.go | 24 ++--- holder.go | 58 ++++++------ holder_test.go | 44 ++++----- index.go | 44 ++++----- index_test.go | 54 +++++------ pilosa.go | 4 +- pql/ast_test.go | 4 +- pql/parser_test.go | 10 +- server/cluster_test.go | 38 ++++---- server/server_test.go | 88 +++++++++--------- server_test.go | 2 +- stats_test.go | 36 +++---- test/cluster.go | 10 +- test/fragment.go | 8 +- test/holder.go | 14 +-- time_test.go | 24 ++--- utils_test.go | 10 +- view_test.go | 8 +- 37 files changed, 552 insertions(+), 552 deletions(-) diff --git a/apimethod_string.go b/apimethod_string.go index ce119196f..a2b934e0f 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -4,7 +4,7 @@ package pilosa import "fmt" -const _apiMethod_name = "apiClusterMessageapiCreateFrameapiCreateIndexapiDeleteFrameapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews" var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 151, 160, 174, 182, 198, 216, 224, 244, 257, 271, 288, 301, 321, 329} diff --git a/client.go b/client.go index ae536f2c4..81f750a39 100644 --- a/client.go +++ b/client.go @@ -542,7 +542,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde u := nodePathToURL(node, "/export") u.RawQuery = url.Values{ "index": {index}, - "frame": {field}, + "field": {field}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode() @@ -585,7 +585,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, field s u := nodePathToURL(node, "/fragment/data") u.RawQuery = url.Values{ "index": {index}, - "frame": {field}, + "field": {field}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode() @@ -622,7 +622,7 @@ func (c *InternalHTTPClient) CreateField(ctx context.Context, index, field strin } // Encode query request. - buf, err := json.Marshal(&postFrameRequest{ + buf, err := json.Marshal(&postFieldRequest{ Options: opt, }) if err != nil { @@ -630,7 +630,7 @@ func (c *InternalHTTPClient) CreateField(ctx context.Context, index, field strin } // Create URL & HTTP request. - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, field)) + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s", index, field)) req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { return errors.Wrap(err, "creating request") @@ -670,7 +670,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, field st u := uriPathToURL(c.defaultURI, "/fragment/blocks") u.RawQuery = url.Values{ "index": {index}, - "frame": {field}, + "field": {field}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode() @@ -795,10 +795,10 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b // RowAttrDiff returns data from differing blocks on a remote host. func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, field)) + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s/attr/diff", index, field)) // Encode request. - buf, err := json.Marshal(postFrameAttrDiffRequest{Blocks: blks}) + buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks}) if err != nil { return nil, errors.Wrap(err, "marshaling") } @@ -828,7 +828,7 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, field strin } // Decode response object. - var rsp postFrameAttrDiffResponse + var rsp postFieldAttrDiffResponse if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { return nil, errors.Wrap(err, "decoding") } diff --git a/client_test.go b/client_test.go index 9091c8200..d2e194805 100644 --- a/client_test.go +++ b/client_test.go @@ -145,7 +145,7 @@ func TestClient_MultiNode(t *testing.T) { topN := 4 queryRequest := &internal.QueryRequest{ - Query: fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN), + Query: fmt.Sprintf(`TopN(field="%s", n=%d)`, "f", topN), Remote: false, } result, err := client[0].Query(context.Background(), "i", queryRequest) @@ -254,7 +254,7 @@ func TestClient_ImportValue(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frame, err := index.CreateFieldIfNotExists(fldName, fo) + field, err := index.CreateFieldIfNotExists(fldName, fo) if err != nil { t.Fatal(err) } @@ -276,7 +276,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Sum. - sum, cnt, err := frame.Sum(nil, fldName) + sum, cnt, err := field.Sum(nil, fldName) if err != nil { t.Fatal(err) } @@ -285,7 +285,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Min. - min, cnt, err := frame.Min(nil, fldName) + min, cnt, err := field.Min(nil, fldName) if err != nil { t.Fatal(err) } @@ -294,11 +294,11 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Min with Filter. - filter, err := frame.Range(fldName, pql.GT, 40) + filter, err := field.Range(fldName, pql.GT, 40) if err != nil { t.Fatal(err) } - min, cnt, err = frame.Min(filter, fldName) + min, cnt, err = field.Min(filter, fldName) if err != nil { t.Fatal(err) } @@ -307,7 +307,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Max. - max, cnt, err := frame.Max(nil, fldName) + max, cnt, err := field.Max(nil, fldName) if err != nil { t.Fatal(err) } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 4b3a159f8..f97775b96 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -45,13 +45,13 @@ func TestFragCombos(t *testing.T) { tests := []struct { idx string maxSlice uint64 - frameViews viewsByField + fieldViews viewsByField expected fragsByHost }{ { idx: "i", maxSlice: uint64(2), - frameViews: viewsByField{"f": []string{"v1", "v2"}}, + fieldViews: viewsByField{"f": []string{"v1", "v2"}}, expected: fragsByHost{ "node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, "node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}}, @@ -60,7 +60,7 @@ func TestFragCombos(t *testing.T) { { idx: "foo", maxSlice: uint64(3), - frameViews: viewsByField{"f": []string{"v0"}}, + fieldViews: viewsByField{"f": []string{"v0"}}, expected: fragsByHost{ "node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, "node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}}, @@ -69,7 +69,7 @@ func TestFragCombos(t *testing.T) { } for _, test := range tests { - actual := c.fragCombos(test.idx, test.maxSlice, test.frameViews) + actual := c.fragCombos(test.idx, test.maxSlice, test.fieldViews) if !reflect.DeepEqual(actual, test.expected) { t.Errorf("expected: %v, but got: %v", test.expected, actual) } @@ -145,23 +145,23 @@ func TestFragSources(t *testing.T) { c5.addNodeBasicSorted(node3) idx := newIndexWithTempPath("i") - frame, err := idx.CreateFieldIfNotExists("f", FieldOptions{}) + field, err := idx.CreateFieldIfNotExists("f", FieldOptions{}) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 101, nil) + _, err = field.SetBit("standard", 1, 101, nil) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 1300000, nil) + _, err = field.SetBit("standard", 1, 1300000, nil) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 2600000, nil) + _, err = field.SetBit("standard", 1, 2600000, nil) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 3900000, nil) + _, err = field.SetBit("standard", 1, 3900000, nil) if err != nil { t.Fatal(err) } diff --git a/cluster_test.go b/cluster_test.go index c0de34250..a977b3536 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -416,7 +416,7 @@ func TestCluster_ResizeStates(t *testing.T) { } // Add Bit Data to node0. - if err := tc.CreateFrame("i", "f", FieldOptions{}); err != nil { + if err := tc.CreateField("i", "f", FieldOptions{}); err != nil { t.Fatal(err) } tc.SetBit("i", "f", "standard", 1, 101, nil) @@ -424,8 +424,8 @@ func TestCluster_ResizeStates(t *testing.T) { // Before starting the resize, get the CheckSum to use for // comparison later. - node0Frame := node0.Holder.Field("i", "f") - node0View := node0Frame.View("standard") + node0Field := node0.Holder.Field("i", "f") + node0View := node0Field.View("standard") node0Fragment := node0View.Fragment(1) node0Checksum := node0Fragment.Checksum() @@ -453,8 +453,8 @@ func TestCluster_ResizeStates(t *testing.T) { // Bits // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. - node1Frame := node1.Holder.Field("i", "f") - node1View := node1Frame.View("standard") + node1Field := node1.Holder.Field("i", "f") + node1View := node1Field.View("standard") node1Fragment := node1View.Fragment(1) // Ensure checksums are the same. diff --git a/cmd/export.go b/cmd/export.go index 951c4a0a9..9613d9544 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -52,7 +52,7 @@ The file does not contain any headers. flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export") - flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export") + flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export") flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout") ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.SkipVerify) diff --git a/cmd/export_test.go b/cmd/export_test.go index 2b30116c6..6f7a49b8f 100644 --- a/cmd/export_test.go +++ b/cmd/export_test.go @@ -37,13 +37,13 @@ func TestExportConfig(t *testing.T) { env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` index = "myindex" -frame = "f1" +field = "f1" `, validation: func() error { v := validator{} v.Check(cmd.Exporter.Host, "localhost:12345") v.Check(cmd.Exporter.Index, "myindex") - v.Check(cmd.Exporter.Frame, "f1") + v.Check(cmd.Exporter.Field, "f1") v.Check(cmd.Exporter.Path, "/somefile") return v.Error() }, diff --git a/cmd/import.go b/cmd/import.go index ea489f59d..db5cebf8d 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command importCmd := &cobra.Command{ Use: "import", Short: "Bulk load data into pilosa.", - Long: `Bulk imports one or more CSV files to a host's index and frame. The data + Long: `Bulk imports one or more CSV files to a host's index and field. The data of the CSV file are grouped by slice for the most efficient import. The format of the CSV file is: @@ -54,14 +54,14 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags := importCmd.Flags() flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") - flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.") + flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") flags.BoolVar(&Importer.StringKeys, "string-keys", false, "Treat payload as string keys.") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") - flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") - flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked") - flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame") + flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Time quantum for the field") + flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Cache type for the field; valid values: none, lru, ranked") + flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Cache size for the field") ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify) return importCmd diff --git a/cmd/import_test.go b/cmd/import_test.go index 8117d46fa..e8e5d61f4 100644 --- a/cmd/import_test.go +++ b/cmd/import_test.go @@ -37,13 +37,13 @@ func TestImportConfig(t *testing.T) { env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` index = "myindex" -frame = "f1" +field = "f1" `, validation: func() error { v := validator{} v.Check(cmd.Importer.Host, "localhost:12345") v.Check(cmd.Importer.Index, "myindex") - v.Check(cmd.Importer.Frame, "f1") + v.Check(cmd.Importer.Field, "f1") return v.Error() }, }, diff --git a/ctl/export.go b/ctl/export.go index 988403272..090904807 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -30,9 +30,9 @@ type ExportCommand struct { // Remote host and port. Host string - // Name of the index & frame to export from. + // Name of the index & field to export from. Index string - Frame string + Field string // Filename to export to. Path string @@ -57,7 +57,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Validate arguments. if cmd.Index == "" { return pilosa.ErrIndexRequired - } else if cmd.Frame == "" { + } else if cmd.Field == "" { return pilosa.ErrFieldRequired } @@ -89,7 +89,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Export each slice. for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil { + if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, slice, w); err != nil { return errors.Wrap(err, "exporting") } } diff --git a/ctl/export_test.go b/ctl/export_test.go index 6948b810f..5e87334d9 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -59,10 +59,10 @@ func TestExportCommand_Run(t *testing.T) { cm.Host = s.Host() http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(""))) cm.Index = "i" - cm.Frame = "f" + cm.Field = "f" if err := cm.Run(context.Background()); err != nil { t.Fatalf("Export Run doesn't work: %s", err) } diff --git a/ctl/import.go b/ctl/import.go index 0bc3d093c..9383c300f 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -35,13 +35,13 @@ type ImportCommand struct { // Destination host and port. Host string `json:"host"` - // Name of the index & frame to import into. + // Name of the index & field to import into. Index string `json:"index"` - Frame string `json:"frame"` + Field string `json:"field"` - // Options for index & frame to be created if they don't exist + // Options for index & field to be created if they don't exist IndexOptions pilosa.IndexOptions - FrameOptions pilosa.FieldOptions + FieldOptions pilosa.FieldOptions // CreateSchema ensures the schema exists before import CreateSchema bool @@ -80,10 +80,10 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // Validate arguments. - // Index and frame are validated early before the files are parsed. + // Index and field are validated early before the files are parsed. if cmd.Index == "" { return pilosa.ErrIndexRequired - } else if cmd.Frame == "" { + } else if cmd.Field == "" { return pilosa.ErrFieldRequired } else if len(cmd.Paths) == 0 { return errors.New("path required") @@ -102,17 +102,17 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } } - // Determine the frame type in order to correctly handle the input data. - frameType := pilosa.DefaultFieldType + // Determine the field type in order to correctly handle the input data. + fieldType := pilosa.DefaultFieldType schema, err := cmd.Client.Schema(ctx) if err != nil { return errors.Wrap(err, "getting schema") } for _, index := range schema { if index.Name == cmd.Index { - for _, frame := range index.Fields { - if frame.Name == cmd.Frame { - frameType = frame.Options.Type + for _, field := range index.Fields { + if field.Name == cmd.Field { + fieldType = field.Options.Type } } } @@ -121,7 +121,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { // Import each path and import by slice. for _, path := range cmd.Paths { logger.Printf("parsing: %s", path) - if err := cmd.importPath(ctx, frameType, path); err != nil { + if err := cmd.importPath(ctx, fieldType, path); err != nil { return err } } @@ -134,17 +134,17 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { if err != nil { return fmt.Errorf("Error Creating Index: %s", err) } - err = cmd.Client.EnsureField(ctx, cmd.Index, cmd.Frame, cmd.FrameOptions) + err = cmd.Client.EnsureField(ctx, cmd.Index, cmd.Field, cmd.FieldOptions) if err != nil { - return fmt.Errorf("Error Creating Frame: %s", err) + return fmt.Errorf("Error Creating Field: %s", err) } return nil } // importPath parses a path into bits and imports it to the server. -func (cmd *ImportCommand) importPath(ctx context.Context, frameType, path string) error { - // If frameType is `int`, treat the import data as values to be range-encoded. - if frameType == pilosa.FieldTypeInt { +func (cmd *ImportCommand) importPath(ctx context.Context, fieldType, path string) error { + // If fieldType is `int`, treat the import data as values to be range-encoded. + if fieldType == pilosa.FieldTypeInt { return cmd.bufferValues(ctx, path) } else { if cmd.StringKeys { @@ -254,7 +254,7 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err } logger.Printf("importing slice: %d, n=%d", slice, len(chunk)) - if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, chunk); err != nil { + if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, slice, chunk); err != nil { return errors.Wrap(err, "importing") } } @@ -351,7 +351,7 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er // TODO: does it help to sort the rowKeys? logger.Printf("importing keys: n=%d", len(bits)) - if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil { + if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Field, bits); err != nil { return errors.Wrap(err, "importing keys") } @@ -448,7 +448,7 @@ func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldV } logger.Printf("importing slice: %d, n=%d", slice, len(vals)) - if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, slice, vals); err != nil { + if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, slice, vals); err != nil { return errors.Wrap(err, "importing values") } } diff --git a/ctl/import_test.go b/ctl/import_test.go index 9b6dd3c6e..ba8a9dc6e 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -43,7 +43,7 @@ func TestImportCommand_Validation(t *testing.T) { t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err) } - cm.Frame = "f" + cm.Field = "f" err = cm.Run(context.Background()) if err.Error() != "path required" { t.Fatalf("Command not working, expect: %s, actual: '%s'", "path required", err) @@ -73,7 +73,7 @@ func TestImportCommand_Run(t *testing.T) { cm.Host = s.Host() cm.Index = "i" - cm.Frame = "f" + cm.Field = "f" cm.CreateSchema = true cm.Paths = []string{file.Name()} err = cm.Run(ctx) @@ -106,10 +106,10 @@ func TestImportCommand_RunValue(t *testing.T) { cm.Host = s.Host() http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) cm.Index = "i" - cm.Frame = "f" + cm.Field = "f" cm.Paths = []string{file.Name()} err = cm.Run(ctx) if err != nil { @@ -133,7 +133,7 @@ func TestImportCommand_InvalidFile(t *testing.T) { cm := NewImportCommand(stdin, stdout, stderr) cm.Host = s.Host() cm.Index = "i" - cm.Frame = "f" + cm.Field = "f" file, err := ioutil.TempFile("", "import.csv") file.Write([]byte("a,2\n3,5\n5,6")) if err != nil { diff --git a/executor.go b/executor.go index d1e40d808..411039d40 100644 --- a/executor.go +++ b/executor.go @@ -26,9 +26,9 @@ import ( "github.com/pkg/errors" ) -// DefaultFrame is the frame used if one is not specified. +// DefaultField is the field used if one is not specified. const ( - DefaultFrame = "general" + DefaultField = "general" // MinThreshold is the lowest count to use in a Top-N operation when // looking for additional id/count pairs. @@ -174,8 +174,8 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { // executeSum executes a Sum() call. func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { - if frame := c.Args["frame"]; frame == "" { - return ValCount{}, errors.New("Sum(): frame required") + if field := c.Args["field"]; field == "" { + return ValCount{}, errors.New("Sum(): field required") } if len(c.Children) > 1 { @@ -207,8 +207,8 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl // executeMin executes a Min() call. func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { - if frame := c.Args["frame"]; frame == "" { - return ValCount{}, errors.New("Min(): frame required") + if field := c.Args["field"]; field == "" { + return ValCount{}, errors.New("Min(): field required") } if len(c.Children) > 1 { @@ -240,8 +240,8 @@ func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, sl // executeMax executes a Max() call. func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { - if frame := c.Args["frame"]; frame == "" { - return ValCount{}, errors.New("Max(): frame required") + if field := c.Args["field"]; field == "" { + return ValCount{}, errors.New("Max(): field required") } if len(c.Children) > 1 { @@ -312,8 +312,8 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else if err != nil { return nil, err } else { - frame, _ := c.Args["frame"].(string) - if fr := idx.Field(frame); fr != nil { + field, _ := c.Args["field"].(string) + if fr := idx.Field(field); fr != nil { rowID, _, err := c.UintArg(rowLabel) if err != nil { return nil, errors.Wrap(err, "getting row") @@ -367,19 +367,19 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq filter = row } - frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) - frame := e.Holder.Field(index, frameName) - if frame == nil { + field := e.Holder.Field(index, fieldName) + if field == nil { return ValCount{}, nil } - bsig := frame.bsiGroup(frameName) + bsig := field.bsiGroup(fieldName) if bsig == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if fragment == nil { return ValCount{}, nil } @@ -405,19 +405,19 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal filter = row } - frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) - frame := e.Holder.Field(index, frameName) - if frame == nil { + field := e.Holder.Field(index, fieldName) + if field == nil { return ValCount{}, nil } - bsig := frame.bsiGroup(frameName) + bsig := field.bsiGroup(fieldName) if bsig == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if fragment == nil { return ValCount{}, nil } @@ -443,19 +443,19 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal filter = row } - frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) - frame := e.Holder.Field(index, frameName) - if frame == nil { + field := e.Holder.Field(index, fieldName) + if field == nil { return ValCount{}, nil } - bsig := frame.bsiGroup(frameName) + bsig := field.bsiGroup(fieldName) if bsig == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if fragment == nil { return ValCount{}, nil } @@ -538,7 +538,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C // executeTopNSlice executes a TopN call for a single slice. func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) { - frame, _ := c.Args["frame"].(string) + field, _ := c.Args["field"].(string) n, _, err := c.UintArg("n") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) @@ -570,12 +570,12 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca return nil, errors.New("TopN() can only have one input bitmap") } - // Set default frame. - if frame == "" { - frame = DefaultFrame + // Set default field. + if field == "" { + field = DefaultField } - f := e.Holder.Fragment(index, frame, ViewStandard, slice) + f := e.Holder.Fragment(index, field, ViewStandard, slice) if f == nil { return nil, nil } @@ -627,12 +627,12 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. return nil, ErrIndexNotFound } - // Fetch frame & row label based on argument. - frame, _ := c.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame + // Fetch field & row label based on argument. + field, _ := c.Args["field"].(string) + if field == "" { + field = DefaultField } - f := e.Holder.Field(index, frame) + f := e.Holder.Field(index, field) if f == nil { return nil, ErrFieldNotFound } @@ -645,7 +645,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel) } - frag := e.Holder.Fragment(index, frame, ViewStandard, slice) + frag := e.Holder.Fragment(index, field, ViewStandard, slice) if frag == nil { return NewRow(), nil } @@ -681,10 +681,10 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return e.executeBSIGroupRangeSlice(ctx, index, c, slice) } - // Parse frame, use default if unset. - frame, _ := c.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame + // Parse field, use default if unset. + field, _ := c.Args["field"].(string) + if field == "" { + field = DefaultField } // Retrieve column label. @@ -693,8 +693,8 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return nil, ErrIndexNotFound } - // Retrieve base frame. - f := idx.Field(frame) + // Retrieve base field. + f := idx.Field(field) if f == nil { return nil, ErrFieldNotFound } @@ -734,10 +734,10 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return &Row{}, nil } - // Union bitmaps across all time-based subframes. + // Union bitmaps across all time-based views. row := &Row{} for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) { - f := e.Holder.Fragment(index, frame, view, slice) + f := e.Holder.Fragment(index, field, view, slice) if f == nil { continue } @@ -757,17 +757,17 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // Extract conditional. - var frameName string + var fieldName string var cond *pql.Condition for k, v := range c.Args { vv, ok := v.(*pql.Condition) if !ok { return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v) } - frameName, cond = k, vv + fieldName, cond = k, vv } - f := e.Holder.Field(index, frameName) + f := e.Holder.Field(index, fieldName) if f == nil { return nil, ErrFieldNotFound } @@ -782,13 +782,13 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, // Handle `!= null`. if cond.Op == pql.NEQ && cond.Value == nil { // Find bsiGroup. - bsig := f.bsiGroup(frameName) + bsig := f.bsiGroup(fieldName) if bsig == nil { return nil, ErrBSIGroupNotFound } // Retrieve fragment. - frag := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if frag == nil { return NewRow(), nil } @@ -808,11 +808,11 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // The reason we don't just call: - // return f.RangeBetween(frameName, predicates[0], predicates[1]) + // return f.RangeBetween(fieldName, predicates[0], predicates[1]) // here is because we need the call to be slice-specific. // Find bsiGroup. - bsig := f.bsiGroup(frameName) + bsig := f.bsiGroup(fieldName) if bsig == nil { return nil, ErrBSIGroupNotFound } @@ -823,7 +823,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // Retrieve fragment. - frag := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if frag == nil { return NewRow(), nil } @@ -845,7 +845,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // Find bsiGroup. - bsig := f.bsiGroup(frameName) + bsig := f.bsiGroup(fieldName) if bsig == nil { return nil, ErrBSIGroupNotFound } @@ -856,7 +856,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // Retrieve fragment. - frag := e.Holder.Fragment(index, frameName, viewBSIGroupPrefix+frameName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if frag == nil { return NewRow(), nil } @@ -949,17 +949,17 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, // executeClearBit executes a ClearBit() call. func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - frame, ok := c.Args["frame"].(string) + field, ok := c.Args["field"].(string) if !ok { - return false, errors.New("ClearBit() frame required") + return false, errors.New("ClearBit() field required") } - // Retrieve frame. + // Retrieve field. idx := e.Holder.Index(index) if idx == nil { return false, ErrIndexNotFound } - f := idx.Field(frame) + f := idx.Field(field) if f == nil { return false, ErrFieldNotFound } @@ -1014,17 +1014,17 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql // executeSetBit executes a SetBit() call. func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - frame, ok := c.Args["frame"].(string) + field, ok := c.Args["field"].(string) if !ok { - return false, errors.New("SetBit() field required: frame") + return false, errors.New("SetBit() field required: field") } - // Retrieve frame. + // Retrieve field. idx := e.Holder.Index(index) if idx == nil { return false, ErrIndexNotFound } - f := idx.Field(frame) + f := idx.Field(field) if f == nil { return false, ErrFieldNotFound } @@ -1101,27 +1101,27 @@ func (e *Executor) executeSetValue(ctx context.Context, index string, c *pql.Cal // Copy args and remove reserved fields. args := pql.CopyArgs(c.Args) - // While frame could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. - // Also, if we ever need to make ColumnAttrs frame-specific, then having this reserved word prevents backward incompatibility. + // While field could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. + // Also, if we ever need to make ColumnAttrs field-specific, then having this reserved word prevents backward incompatibility. delete(args, columnLabel) // Set values. for name, value := range args { - // Retrieve frame. - frame := e.Holder.Field(index, name) - if frame == nil { + // Retrieve field. + field := e.Holder.Field(index, name) + if field == nil { return ErrFieldNotFound } switch value := value.(type) { case int64: - if _, err := frame.SetValue(columnID, value); err != nil { + if _, err := field.SetValue(columnID, value); err != nil { return err } default: return ErrInvalidBSIGroupValueType } - frame.Stats.Count("SetValue", 1, 1.0) + field.Stats.Count("SetValue", 1, 1.0) } // Do not forward call if this is already being forwarded. @@ -1151,14 +1151,14 @@ func (e *Executor) executeSetValue(ctx context.Context, index string, c *pql.Cal // executeSetRowAttrs executes a SetRowAttrs() call. func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { - frameName, ok := c.Args["frame"].(string) + fieldName, ok := c.Args["field"].(string) if !ok { - return errors.New("SetRowAttrs() frame required") + return errors.New("SetRowAttrs() field required") } - // Retrieve frame. - frame := e.Holder.Field(index, frameName) - if frame == nil { + // Retrieve field. + field := e.Holder.Field(index, fieldName) + if field == nil { return ErrFieldNotFound } @@ -1172,14 +1172,14 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, "frame") + delete(attrs, "field") delete(attrs, rowLabel) // Set attributes. - if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { + if err := field.RowAttrStore().SetAttrs(rowID, attrs); err != nil { return err } - frame.Stats.Count("SetRowAttrs", 1, 1.0) + field.Stats.Count("SetRowAttrs", 1, 1.0) // Do not forward call if this is already being forwarded. if opt.Remote { @@ -1208,16 +1208,16 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. // executeBulkSetRowAttrs executes a set of SetRowAttrs() calls. func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) { - // Collect attributes by frame/id. + // Collect attributes by field/id. m := make(map[string]map[uint64]map[string]interface{}) for _, c := range calls { - frame, ok := c.Args["frame"].(string) + field, ok := c.Args["field"].(string) if !ok { - return nil, errors.New("SetRowAttrs() frame required") + return nil, errors.New("SetRowAttrs() field required") } - // Retrieve frame. - f := e.Holder.Field(index, frame) + // Retrieve field. + f := e.Holder.Field(index, field) if f == nil { return nil, ErrFieldNotFound } @@ -1231,20 +1231,20 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, "frame") + delete(attrs, "field") delete(attrs, rowLabel) - // Create frame group, if not exists. - frameMap := m[frame] - if frameMap == nil { - frameMap = make(map[uint64]map[string]interface{}) - m[frame] = frameMap + // Create field group, if not exists. + fieldMap := m[field] + if fieldMap == nil { + fieldMap = make(map[uint64]map[string]interface{}) + m[field] = fieldMap } // Set or merge attributes. - attr := frameMap[rowID] + attr := fieldMap[rowID] if attr == nil { - frameMap[rowID] = cloneAttrs(attrs) + fieldMap[rowID] = cloneAttrs(attrs) } else { for k, v := range attrs { attr[k] = v @@ -1252,19 +1252,19 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal } } - // Bulk insert attributes by frame. - for name, frameMap := range m { - // Retrieve frame. - frame := e.Holder.Field(index, name) - if frame == nil { + // Bulk insert attributes by field. + for name, fieldMap := range m { + // Retrieve field. + field := e.Holder.Field(index, name) + if field == nil { return nil, ErrFieldNotFound } // Set attributes. - if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil { + if err := field.RowAttrStore().SetBulkAttrs(fieldMap); err != nil { return nil, err } - frame.Stats.Count("SetRowAttrs", 1, 1.0) + field.Stats.Count("SetRowAttrs", 1, 1.0) } // Do not forward call if this is already being forwarded. @@ -1309,7 +1309,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, columnLabel) - delete(attrs, "frame") + delete(attrs, "field") // Set attributes. if err := idx.ColumnAttrStore().SetAttrs(col, attrs); err != nil { diff --git a/executor_test.go b/executor_test.go index bd965c325..01c99e9c5 100644 --- a/executor_test.go +++ b/executor_test.go @@ -42,9 +42,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -52,7 +52,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) @@ -61,7 +61,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit column attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) @@ -70,7 +70,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit row attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) @@ -91,9 +91,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -225,7 +225,7 @@ func TestExecutor_Execute_Count(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { t.Fatalf("unexpected n: %d", res[0]) @@ -243,7 +243,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -254,7 +254,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if n := f.Row(11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -269,7 +269,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Create frames. + // Create felds. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, @@ -347,7 +347,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Create frames. + // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) @@ -356,18 +356,18 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { } // Set two attrs on f/10. - // Also set attrs on other bitmaps and frames to test isolation. + // Also set attrs on other bitmaps and fields to test isolation. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, frame=f, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, field=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=xxx, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } @@ -393,15 +393,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=1) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(frame=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(frame=f, row=10, col=0) - SetBit(frame=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=other, row=0, col=0) + SetBit(field=f, row=0, col=0) + SetBit(field=f, row=0, col=1) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(field=f, row=10, col=0) + SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=other, row=0, col=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -411,7 +411,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() t.Run("Standard", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ {ID: 0, Count: 5}, @@ -436,7 +436,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { // Execute query. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 4}, @@ -470,7 +470,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { // Execute query. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, @@ -505,7 +505,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { // Execute query. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, field=other), field=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, @@ -529,7 +529,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -552,7 +552,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,frame=f),frame="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,field=f),field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -585,11 +585,11 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=x, row=0, col=0) - SetBit(frame=x, row=0, col=3) - SetBit(frame=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetBit(frame=x, row=1, col=1) - SetBit(frame=x, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(field=x, row=0, col=0) + SetBit(field=x, row=0, col=3) + SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(field=x, row=1, col=1) + SetBit(field=x, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) SetValue(f=20, col=0) SetValue(f=-5, col=1) @@ -610,16 +610,16 @@ func TestExecutor_Execute_MinMax(t *testing.T) { cnt int64 }{ {filter: ``, exp: -5, cnt: 2}, - {filter: `Bitmap(frame=x, row=0)`, exp: 10, cnt: 1}, - {filter: `Bitmap(frame=x, row=1)`, exp: -5, cnt: 1}, - {filter: `Bitmap(frame=x, row=2)`, exp: 40, cnt: 1}, + {filter: `Bitmap(field=x, row=0)`, exp: 10, cnt: 1}, + {filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1}, } for i, tt := range tests { var pql string if tt.filter == "" { - pql = `Min(frame=f)` + pql = `Min(field=f)` } else { - pql = fmt.Sprintf(`Min(%s, frame=f)`, tt.filter) + pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) @@ -636,16 +636,16 @@ func TestExecutor_Execute_MinMax(t *testing.T) { cnt int64 }{ {filter: ``, exp: 60, cnt: 1}, - {filter: `Bitmap(frame=x, row=0)`, exp: 60, cnt: 1}, - {filter: `Bitmap(frame=x, row=1)`, exp: -5, cnt: 1}, - {filter: `Bitmap(frame=x, row=2)`, exp: 40, cnt: 1}, + {filter: `Bitmap(field=x, row=0)`, exp: 60, cnt: 1}, + {filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1}, } for i, tt := range tests { var pql string if tt.filter == "" { - pql = `Max(frame=f)` + pql = `Max(field=f)` } else { - pql = fmt.Sprintf(`Max(%s, frame=f)`, tt.filter) + pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) @@ -696,8 +696,8 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=x, row=0, col=0) - SetBit(frame=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(field=x, row=0, col=0) + SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) SetValue(foo=20, col=0) SetValue(bar=2000, col=0) @@ -711,7 +711,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { } t.Run("NoFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(field=foo)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -719,7 +719,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=x, row=0), frame=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(field=x, row=0), field=foo)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -736,7 +736,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { // Create index. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - // Create frame. + // Create field. if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeTime, TimeQuantum: pilosa.TimeQuantum("YMDH"), @@ -746,22 +746,22 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { // Set columns. if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00") - SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00") - SetBit(frame=f, row=1, col=4, timestamp="2000-01-02T00:00") - SetBit(frame=f, row=1, col=5, timestamp="2000-02-01T00:00") - SetBit(frame=f, row=1, col=6, timestamp="2001-01-01T00:00") - SetBit(frame=f, row=1, col=7, timestamp="2002-01-01T02:00") + SetBit(field=f, row=1, col=2, timestamp="1999-12-31T00:00") + SetBit(field=f, row=1, col=3, timestamp="2000-01-01T00:00") + SetBit(field=f, row=1, col=4, timestamp="2000-01-02T00:00") + SetBit(field=f, row=1, col=5, timestamp="2000-02-01T00:00") + SetBit(field=f, row=1, col=6, timestamp="2001-01-01T00:00") + SetBit(field=f, row=1, col=7, timestamp="2002-01-01T02:00") - SetBit(frame=f, row=1, col=2, timestamp="1999-12-30T00:00") - SetBit(frame=f, row=1, col=2, timestamp="2002-02-01T00:00") - SetBit(frame=f, row=10, col=2, timestamp="2001-01-01T00:00") + SetBit(field=f, row=1, col=2, timestamp="1999-12-30T00:00") + SetBit(field=f, row=1, col=2, timestamp="2002-02-01T00:00") + SetBit(field=f, row=10, col=2, timestamp="2001-01-01T00:00") `), nil, nil); err != nil { t.Fatal(err) } t.Run("Standard", func(t *testing.T) { - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, field=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) @@ -817,8 +817,8 @@ func TestExecutor_Execute_Range(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(field=f, row=0, col=0) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) SetValue(foo=20, col=50) SetValue(bar=2000, col=50) @@ -944,8 +944,8 @@ func TestExecutor_Execute_Range(t *testing.T) { } }) - t.Run("ErrFrameNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(bad_frame >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { + t.Run("ErrFieldNotFound", func(t *testing.T) { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { t.Fatal(err) } }) @@ -969,7 +969,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Bitmap(frame="f", row=10)` { + } else if query.String() != `Bitmap(field="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{1}) { t.Fatalf("unexpected slices: %+v", slices) @@ -992,7 +992,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) { t.Fatalf("unexpected columns: %+v", columns) @@ -1027,7 +1027,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(12) { t.Fatalf("unexpected n: %d", res[0]) @@ -1055,7 +1055,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(col=2, frame="f", row=10)` { + } else if query.String() != `SetBit(col=2, field="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1067,13 +1067,13 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { defer hldr.Close() s.Handler.API.Holder = hldr.Holder - // Create frame. + // Create field. if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2)`), nil, nil); err != nil { t.Fatal(err) } @@ -1107,7 +1107,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(col=2, frame="f", row=10, timestamp="2016-12-11T10:09")` { + } else if query.String() != `SetBit(col=2, field="f", row=10, timestamp="2016-12-11T10:09")` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1119,7 +1119,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { defer hldr.Close() s.Handler.API.Holder = hldr.Holder - // Create frame. + // Create field. if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := f.SetTimeQuantum("Y"); err != nil { @@ -1127,7 +1127,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } @@ -1168,11 +1168,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // slices and a second time to get the counts for a set of bitmaps. switch remoteExecN { case 0: - if query.String() != `TopN(frame="f", n=3)` { + if query.String() != `TopN(field="f", n=3)` { t.Fatalf("unexpected query(0): %s", query.String()) } case 1: - if query.String() != `TopN(frame="f", ids=[0,10,30], n=3)` { + if query.String() != `TopN(field="f", ids=[0,10,30], n=3)` { t.Fatalf("unexpected query(1): %s", query.String()) } default: @@ -1196,7 +1196,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, @@ -1218,8 +1218,8 @@ func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { } } -// Ensure SetColumnAttrs doesn't save `frame` as an attribute -func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { +// Ensure SetColumnAttrs doesn't save `field` as an attribute +func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -1229,12 +1229,12 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { } e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // SetColumnAttrs call should exclude the frame attribute - _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=10)"), nil, nil) + // SetColumnAttrs call should exclude the field attribute + _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=10)"), nil, nil) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', col=10, foo='bar')"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(field='f', col=10, foo='bar')"), nil, nil) if err != nil { t.Fatal(err) } @@ -1246,8 +1246,8 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { t.Fatalf("%#v != %#v", targetAttrs, attrs) } - // SetColumnAttrs call should not break if frame is not specified - _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=20)"), nil, nil) + // SetColumnAttrs call should not break if field is not specified + _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=20)"), nil, nil) if err != nil { t.Fatal(err) } diff --git a/fragment.go b/fragment.go index 94232bfa0..1dfa9d09f 100644 --- a/fragment.go +++ b/fragment.go @@ -1904,11 +1904,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Only sync the standard block. for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Field(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.Field(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) count++ } for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Field(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.Field(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) count++ } diff --git a/fragment_test.go b/fragment_test.go index 7f140e907..747602348 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -753,14 +753,14 @@ func TestFragment_TopN_CacheSize(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame. - frame, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) + // Create field. + field, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) if err != nil { t.Fatal(err) } // Create view. - view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard) + view, err := field.CreateViewIfNotExists(pilosa.ViewStandard) if err != nil { t.Fatal(err) } @@ -923,14 +923,14 @@ func TestFragment_RankCache_Persistence(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame. - frame, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) + // Create field. + field, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) if err != nil { t.Fatal(err) } // Create view. - view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard) + view, err := field.CreateViewIfNotExists(pilosa.ViewStandard) if err != nil { t.Fatal(err) } diff --git a/handler.go b/handler.go index dca880aba..083d40279 100644 --- a/handler.go +++ b/handler.go @@ -62,7 +62,7 @@ var externalPrefixFlag = map[string]bool{ "import": true, "export": true, "index": true, - "frame": true, + "field": true, "nodes": true, "version": true, } @@ -108,10 +108,10 @@ func (h *Handler) populateValidators() { h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") h.validators["GetSliceMax"] = queryValidationSpecRequired() h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") - h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "slice") - h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "slice") - h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "slice") - h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "frame", "slice") + h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "slice") } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { @@ -165,10 +165,10 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") - //router.HandleFunc("/index/{index}/frame", handler.handleGetFrames).Methods("GET") // Not implemented. - router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") - router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") + //router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented. + router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") + router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") + router.HandleFunc("/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") @@ -502,22 +502,22 @@ type postIndexAttrDiffResponse struct { Attrs map[uint64]map[string]interface{} `json:"attrs"` } -// handlePostFrame handles POST /frame request. -func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { +// handlePostField handles POST /field request. +func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] + fieldName := mux.Vars(r)["field"] // Decode request. - var req postFrameRequest + var req postFieldRequest err := json.NewDecoder(r.Body).Decode(&req) if err == io.EOF { - // If no data was provided (EOF), we still create the frame + // If no data was provided (EOF), we still create the field // with default values. } else if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - _, err = h.API.CreateField(r.Context(), indexName, frameName, req.Options) + _, err = h.API.CreateField(r.Context(), indexName, fieldName, req.Options) if err != nil { switch errors.Cause(err) { case ErrIndexNotFound: @@ -530,30 +530,30 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { return } // Encode response. - if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(postFieldResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } } -type _postFrameRequest postFrameRequest +type _postFieldRequest postFieldRequest -// Custom Unmarshal JSON to validate request body when creating a new frame. If there's new FrameOptions, -// adding it to validFrameOptions to make sure the new option is validated, otherwise the request will be failed -func (p *postFrameRequest) UnmarshalJSON(b []byte) error { +// Custom Unmarshal JSON to validate request body when creating a new field. If there's new FieldOptions, +// adding it to validFieldOptions to make sure the new option is validated, otherwise the request will be failed +func (p *postFieldRequest) UnmarshalJSON(b []byte) error { // m is an overflow map used to capture additional, unexpected keys. m := make(map[string]interface{}) if err := json.Unmarshal(b, &m); err != nil { return errors.Wrap(err, "unmarshaling unexpected keys") } - validFrameOptions := getValidOptions(FieldOptions{}) - err := validateOptions(m, validFrameOptions) + validFieldOptions := getValidOptions(FieldOptions{}) + err := validateOptions(m, validFieldOptions) if err != nil { return err } // Unmarshal expected values. - var _p _postFrameRequest + var _p _postFieldRequest if err := json.Unmarshal(b, &_p); err != nil { return errors.Wrap(err, "unmarshalling expected keys") } @@ -574,18 +574,18 @@ func getValidOptions(option interface{}) []string { return validOptions } -type postFrameRequest struct { +type postFieldRequest struct { Options FieldOptions `json:"options"` } -type postFrameResponse struct{} +type postFieldResponse struct{} -// handleDeleteFrame handles DELETE /frame request. -func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { +// handleDeleteField handles DELETE /field request. +func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] + fieldName := mux.Vars(r)["field"] - err := h.API.DeleteField(r.Context(), indexName, frameName) + err := h.API.DeleteField(r.Context(), indexName, fieldName) if err != nil { if errors.Cause(err) == ErrIndexNotFound { if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { @@ -598,26 +598,26 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { } // Encode response. - if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(deleteFieldResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } } -type deleteFrameResponse struct{} +type deleteFieldResponse struct{} -// handlePostFrameAttrDiff handles POST /frame/attr/diff requests. -func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) { +// handlePostFieldAttrDiff handles POST /field/attr/diff requests. +func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] + fieldName := mux.Vars(r)["field"] // Decode request. - var req postFrameAttrDiffRequest + var req postFieldAttrDiffRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - attrs, err := h.API.FieldAttrDiff(r.Context(), indexName, frameName, req.Blocks) + attrs, err := h.API.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks) if err != nil { switch errors.Cause(err) { case ErrFragmentNotFound: @@ -629,18 +629,18 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request } // Encode response. - if err := json.NewEncoder(w).Encode(postFrameAttrDiffResponse{ + if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{ Attrs: attrs, }); err != nil { h.Logger.Printf("response encoding error: %s", err) } } -type postFrameAttrDiffRequest struct { +type postFieldAttrDiffRequest struct { Blocks []AttrBlock `json:"blocks"` } -type postFrameAttrDiffResponse struct { +type postFieldAttrDiffResponse struct { Attrs map[uint64]map[string]interface{} `json:"attrs"` } @@ -839,7 +839,7 @@ func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { // Parse query parameters. q := r.URL.Query() - index, frame := q.Get("index"), q.Get("frame") + index, field := q.Get("index"), q.Get("field") slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) if err != nil { @@ -847,7 +847,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { return } - if err = h.API.ExportCSV(r.Context(), index, frame, slice, w); err != nil { + if err = h.API.ExportCSV(r.Context(), index, field, slice, w); err != nil { switch errors.Cause(err) { case ErrFragmentNotFound: break @@ -915,7 +915,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request return } - blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), slice) + blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), slice) if err != nil { if errors.Cause(err) == ErrFragmentNotFound { http.Error(w, err.Error(), http.StatusNotFound) diff --git a/handler_internal_test.go b/handler_internal_test.go index 9adab16ac..837c456ca 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -55,23 +55,23 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { } } -// Test custom UnmarshalJSON for postFrameRequest object -func TestPostFrameRequestUnmarshalJSON(t *testing.T) { +// Test custom UnmarshalJSON for postFieldRequest object +func TestPostFieldRequestUnmarshalJSON(t *testing.T) { tests := []struct { json string - expected postFrameRequest + expected postFieldRequest err string }{ - {json: `{"options": {}}`, expected: postFrameRequest{Options: FieldOptions{}}}, + {json: `{"options": {}}`, expected: postFieldRequest{Options: FieldOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, {json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"}, - {json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FieldOptions{CacheType: "type"}}}, + {json: `{"options": {"cacheType": "type"}}`, expected: postFieldRequest{Options: FieldOptions{CacheType: "type"}}}, {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, } for _, test := range tests { - actual := &postFrameRequest{} + actual := &postFieldRequest{} err := json.Unmarshal([]byte(test.json), actual) if err != nil { if test.err == "" || test.err != err.Error() { diff --git a/handler_test.go b/handler_test.go index bd6449862..036512de4 100644 --- a/handler_test.go +++ b/handler_test.go @@ -104,8 +104,8 @@ func TestHandler_Schema(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -536,7 +536,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" { @@ -560,7 +560,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusOK { @@ -611,7 +611,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusBadRequest { @@ -694,8 +694,8 @@ func TestHandler_Index_Delete(t *testing.T) { } } -// Ensure handler can delete a frame. -func TestHandler_DeleteFrame(t *testing.T) { +// Ensure handler can delete a field. +func TestHandler_DeleteField(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) @@ -707,13 +707,13 @@ func TestHandler_DeleteFrame(t *testing.T) { h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader(""))) + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/field/f1", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) } else if f := hldr.Index("i0").Field("f1"); f != nil { - t.Fatal("expected nil frame") + t.Fatal("expected nil field") } } @@ -766,8 +766,8 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { } } -// Ensure the handler can return data in differing blocks for a frame. -func TestHandler_Frame_AttrStore_Diff(t *testing.T) { +// Ensure the handler can return data in differing blocks for a field. +func TestHandler_Field_AttrStore_Diff(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -801,7 +801,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { // Send block checksums to determine diff. resp, err := http.Post( - s.URL+"/index/i/frame/meta/attr/diff", + s.URL+"/index/i/field/meta/attr/diff", "application/json", strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), ) diff --git a/holder.go b/holder.go index bbff69927..e8db140ce 100644 --- a/holder.go +++ b/holder.go @@ -209,14 +209,14 @@ func (h *Holder) MaxSlices() map[string]uint64 { return a } -// Schema returns schema information for all indexes, frames, and views. +// Schema returns schema information for all indexes, fields, and views. func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{Name: index.Name()} - for _, frame := range index.Fields() { - fi := &FieldInfo{Name: frame.Name(), Options: frame.Options()} - for _, view := range frame.Views() { + for _, field := range index.Fields() { + fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + for _, view := range field.Views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) } sort.Sort(viewInfoSlice(fi.Views)) @@ -238,16 +238,16 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { if err != nil { return errors.Wrap(err, "creating index") } - // Create frames that don't exist. + // Create fields that don't exist. for _, f := range index.Fields { opt := decodeFieldOptions(f.Meta) - frame, err := idx.CreateFieldIfNotExists(f.Name, *opt) + field, err := idx.CreateFieldIfNotExists(f.Name, *opt) if err != nil { - return errors.Wrap(err, "creating frame") + return errors.Wrap(err, "creating field") } // Create views that don't exist. for _, v := range f.Views { - _, err := frame.CreateViewIfNotExists(v) + _, err := field.CreateViewIfNotExists(v) if err != nil { return errors.Wrap(err, "creating view") } @@ -399,18 +399,18 @@ func (h *Holder) Field(index, name string) *Field { return idx.Field(name) } -// View returns the view for an index, frame, and name. -func (h *Holder) View(index, frame, name string) *View { - f := h.Field(index, frame) +// View returns the view for an index, field, and name. +func (h *Holder) View(index, field, name string) *View { + f := h.Field(index, field) if f == nil { return nil } return f.View(name) } -// Fragment returns the fragment for an index, frame & slice. -func (h *Holder) Fragment(index, frame, view string, slice uint64) *Fragment { - v := h.View(index, frame, view) +// Fragment returns the fragment for an index, field & slice. +func (h *Holder) Fragment(index, field, view string, slice uint64) *Fragment { + v := h.View(index, field, view) if v == nil { return nil } @@ -435,8 +435,8 @@ func (h *Holder) monitorCacheFlush() { func (h *Holder) flushCaches() { for _, index := range h.Indexes() { - for _, frame := range index.Fields() { - for _, view := range frame.Views() { + for _, field := range index.Fields() { + for _, view := range field.Views() { for _, fragment := range view.Fragments() { select { case <-h.closing: @@ -606,9 +606,9 @@ func (s *HolderSyncer) SyncHolder() error { return nil } - // Sync frame row attributes. + // Sync field row attributes. if err := s.syncField(di.Name, fi.Name); err != nil { - return fmt.Errorf("frame sync error: index=%s, frame=%s, err=%s", di.Name, fi.Name, err) + return fmt.Errorf("field sync error: index=%s, field=%s, err=%s", di.Name, fi.Name, err) } for _, vi := range fi.Views { @@ -630,7 +630,7 @@ func (s *HolderSyncer) SyncHolder() error { // Sync fragment if own it. if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil { - return fmt.Errorf("fragment sync error: index=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) + return fmt.Errorf("fragment sync error: index=%s, field=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) } } } @@ -691,20 +691,20 @@ func (s *HolderSyncer) syncIndex(index string) error { // syncField synchronizes field attributes with the rest of the cluster. func (s *HolderSyncer) syncField(index, name string) error { - // Retrieve frame reference. + // Retrieve field reference. f := s.Holder.Field(index, name) if f == nil { return nil } indexTag := fmt.Sprintf("index:%s", index) - frameTag := fmt.Sprintf("frame:%s", name) + fieldTag := fmt.Sprintf("field:%s", name) // Read block checksums. blks, err := f.RowAttrStore().Blocks() if err != nil { return errors.Wrap(err, "getting blocks") } - s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag}) + s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, fieldTag}) // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { @@ -714,13 +714,13 @@ func (s *HolderSyncer) syncField(index, name string) error { // Skip update and recomputation if no attributes have changed. m, err := client.RowAttrDiff(context.Background(), index, name, blks) if err == ErrFieldNotFound { - continue // frame not created remotely yet, skip + continue // field not created remotely yet, skip } else if err != nil { return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { continue } - s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.ID}) + s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, fieldTag, node.ID}) // Update local copy. if err := f.RowAttrStore().SetBulkAttrs(m); err != nil { @@ -738,9 +738,9 @@ func (s *HolderSyncer) syncField(index, name string) error { } // syncFragment synchronizes a fragment with the rest of the cluster. -func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) error { - // Retrieve local frame. - f := s.Holder.Field(index, frame) +func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) error { + // Retrieve local field. + f := s.Holder.Field(index, field) if f == nil { return ErrFieldNotFound } @@ -806,8 +806,8 @@ func (c *HolderCleaner) CleanHolder() error { containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node) // Get the fragments registered in memory. - for _, frame := range index.Fields() { - for _, view := range frame.Views() { + for _, field := range index.Fields() { + for _, view := range field.Views() { for _, fragment := range view.Fragments() { fragSlice := fragment.Slice() // Ignore fragments that should be present. diff --git a/holder_test.go b/holder_test.go index 35e3b8e26..5f41cfab7 100644 --- a/holder_test.go +++ b/holder_test.go @@ -91,7 +91,7 @@ func TestHolder_Open(t *testing.T) { } }) - t.Run("ErrFramePermission", func(t *testing.T) { + t.Run("ErrFieldPermission", func(t *testing.T) { if os.Geteuid() == 0 { t.Skip("Skipping permissions test since user is root.") } @@ -127,11 +127,11 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=loading meta: unmarshaling: unexpected EOF") { + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=loading meta: unmarshaling: unexpected EOF") { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrFrameAttrStoreCorrupt", func(t *testing.T) { + t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) { h := test.MustOpenHolder() defer h.Close() @@ -145,7 +145,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=opening attrstore: opening storage: invalid database") { + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=opening attrstore: opening storage: invalid database") { t.Fatalf("unexpected error: %s", err) } }) @@ -159,9 +159,9 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + } else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -183,9 +183,9 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + } else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -208,9 +208,9 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + } else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) } else if _, err := view.SetBit(0, 0); err != nil { t.Fatal(err) @@ -231,9 +231,9 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + } else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) } else if _, err := view.SetBit(0, 0); err != nil { t.Fatal(err) @@ -257,9 +257,9 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + } else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) } else if _, err := view.SetBit(0, 0); err != nil { t.Fatal(err) @@ -391,11 +391,11 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) cluster.Nodes[1].URI = *uri - // Create frames on nodes. + // Create fields on nodes. for _, hldr := range []*test.Holder{hldr0, hldr1} { - hldr.MustCreateFrameIfNotExists("i", "f") - hldr.MustCreateFrameIfNotExists("i", "f0") - hldr.MustCreateFrameIfNotExists("y", "z") + hldr.MustCreateFieldIfNotExists("i", "f") + hldr.MustCreateFieldIfNotExists("i", "f0") + hldr.MustCreateFieldIfNotExists("y", "z") } // Set data on the local holder. @@ -496,11 +496,11 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) - // Create frames on nodes. + // Create fields on nodes. for _, hldr := range []*test.Holder{hldr0} { - hldr.MustCreateFrameIfNotExists("i", "f") - hldr.MustCreateFrameIfNotExists("i", "f0") - hldr.MustCreateFrameIfNotExists("y", "z") + hldr.MustCreateFieldIfNotExists("i", "f") + hldr.MustCreateFieldIfNotExists("i", "f0") + hldr.MustCreateFieldIfNotExists("y", "z") } // Set data on the local holder. diff --git a/index.go b/index.go index 72b9cc1e7..80a5c2eb8 100644 --- a/index.go +++ b/index.go @@ -28,7 +28,7 @@ import ( "github.com/pkg/errors" ) -// Index represents a container for frames. +// Index represents a container for fields. type Index struct { mu sync.RWMutex path string @@ -107,7 +107,7 @@ func (i *Index) Open() error { } if err := i.openFields(); err != nil { - return errors.Wrap(err, "opening frames") + return errors.Wrap(err, "opening fields") } if err := i.columnAttrStore.Open(); err != nil { @@ -117,7 +117,7 @@ func (i *Index) Open() error { return nil } -// openFields opens and initializes the frames inside the index. +// openFields opens and initializes the fields inside the index. func (i *Index) openFields() error { f, err := os.Open(i.path) if err != nil { @@ -140,7 +140,7 @@ func (i *Index) openFields() error { return ErrName } if err := fld.Open(); err != nil { - return fmt.Errorf("open frame: name=%s, err=%s", fld.Name(), err) + return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err) } i.fields[fld.Name()] = fld } @@ -189,7 +189,7 @@ func (i *Index) saveMeta() error { } */ -// Close closes the index and its frames. +// Close closes the index and its fields. func (i *Index) Close() error { i.mu.Lock() defer i.mu.Unlock() @@ -197,10 +197,10 @@ func (i *Index) Close() error { // Close the attribute store. i.columnAttrStore.Close() - // Close all frames. + // Close all fields. for _, f := range i.fields { if err := f.Close(); err != nil { - return errors.Wrap(err, "closing frame") + return errors.Wrap(err, "closing field") } } i.fields = make(map[string]*Field) @@ -237,7 +237,7 @@ func (i *Index) SetRemoteMaxSlice(newmax uint64) { // FieldPath returns the path to a field in the index. func (i *Index) FieldPath(name string) string { return filepath.Join(i.path, name) } -// Field returns a frame in the index by name. +// Field returns a field in the index by name. func (i *Index) Field(name string) *Field { i.mu.RLock() defer i.mu.RUnlock() @@ -260,10 +260,10 @@ func (i *Index) Fields() []*Field { return a } -// RecalculateCaches recalculates caches on every frame in the index. +// RecalculateCaches recalculates caches on every field in the index. func (i *Index) RecalculateCaches() { - for _, frame := range i.Fields() { - frame.RecalculateCaches() + for _, field := range i.Fields() { + field.RecalculateCaches() } } @@ -272,7 +272,7 @@ func (i *Index) CreateField(name string, opt FieldOptions) (*Field, error) { i.mu.Lock() defer i.mu.Unlock() - // Ensure frame doesn't already exist. + // Ensure field doesn't already exist. if i.fields[name] != nil { return nil, ErrFieldExists } @@ -284,7 +284,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, e i.mu.Lock() defer i.mu.Unlock() - // Find frame in cache first. + // Find field in cache first. if f := i.fields[name]; f != nil { return f, nil } @@ -294,7 +294,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, e func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { if name == "" { - return nil, errors.New("frame name required") + return nil, errors.New("field name required") } else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) { return nil, ErrInvalidCacheType } @@ -304,18 +304,18 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { return nil, errors.Wrap(err, "validating options") } - // Initialize frame. + // Initialize field. f, err := i.newField(i.FieldPath(name), name) if err != nil { return nil, errors.Wrap(err, "initializing") } - // Open frame. + // Open field. if err := f.Open(); err != nil { return nil, errors.Wrap(err, "opening") } - // Apply frame options. + // Apply field options. if err := f.applyOptions(opt); err != nil { f.Close() return nil, errors.Wrap(err, "applying options") @@ -326,7 +326,7 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { return nil, errors.Wrap(err, "saving meta") } - // Add to index's frame lookup. + // Add to index's field lookup. i.fields[name] = f return f, nil @@ -338,7 +338,7 @@ func (i *Index) newField(path, name string) (*Field, error) { return nil, err } f.Logger = i.Logger - f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name)) + f.Stats = i.Stats.WithTags(fmt.Sprintf("field:%s", name)) f.broadcaster = i.broadcaster f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data")) return f, nil @@ -349,18 +349,18 @@ func (i *Index) DeleteField(name string) error { i.mu.Lock() defer i.mu.Unlock() - // Ignore if frame doesn't exist. + // Ignore if field doesn't exist. f := i.field(name) if f == nil { return nil } - // Close frame. + // Close field. if err := f.Close(); err != nil { return errors.Wrap(err, "closing") } - // Delete frame directory. + // Delete field directory. if err := os.RemoveAll(i.FieldPath(name)); err != nil { return errors.Wrap(err, "removing directory") } diff --git a/index_test.go b/index_test.go index 2272fd5bd..6f067f0e7 100644 --- a/index_test.go +++ b/index_test.go @@ -23,40 +23,40 @@ import ( "github.com/pilosa/pilosa/test" ) -// Ensure index can open and retrieve a frame. -func TestIndex_CreateFrameIfNotExists(t *testing.T) { +// Ensure index can open and retrieve a field. +func TestIndex_CreateFieldIfNotExists(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame. + // Create field. f, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } else if f == nil { - t.Fatal("expected frame") + t.Fatal("expected field") } - // Retrieve existing frame. + // Retrieve existing field. other, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } else if f.Field != other.Field { - t.Fatal("frame mismatch") + t.Fatal("field mismatch") } if f.Field != index.Field("f") { - t.Fatal("frame mismatch") + t.Fatal("field mismatch") } } -func TestIndex_CreateFrame(t *testing.T) { - // Ensure time quantum can be set appropriately on a new frame. +func TestIndex_CreateField(t *testing.T) { + // Ensure time quantum can be set appropriately on a new field. t.Run("TimeQuantum", func(t *testing.T) { t.Run("Explicit", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame with explicit quantum. + // Create field with explicit quantum. f, err := index.CreateField("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeTime, TimeQuantum: pilosa.TimeQuantum("YMDH"), @@ -64,18 +64,18 @@ func TestIndex_CreateFrame(t *testing.T) { if err != nil { t.Fatal(err) } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected frame time quantum: %s", q) + t.Fatalf("unexpected field time quantum: %s", q) } }) }) - // Ensure frame can include range columns. + // Ensure field can include range columns. t.Run("BSIFields", func(t *testing.T) { t.Run("OK", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame with schema and verify it exists. + // Create field with schema and verify it exists. if f, err := index.CreateField("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: 10, @@ -95,13 +95,13 @@ func TestIndex_CreateFrame(t *testing.T) { }) // TODO: These errors don't apply here. Instead, we need these tests - // on frame creation FrameOptions validation. + // on field creation FieldOptions validation. /* t.Run("ErrRangeCacheAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + if _, err := index.CreateField("f", pilosa.FieldOptions{ CacheType: pilosa.CacheTypeRanked, }); err != nil { t.Fatal(err) @@ -111,7 +111,7 @@ func TestIndex_CreateFrame(t *testing.T) { t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + if _, err := index.CreateField("f", pilosa.FieldOptions{ CacheType: pilosa.CacheTypeNone, CacheSize: uint32(5), }); err != nil { @@ -119,11 +119,11 @@ func TestIndex_CreateFrame(t *testing.T) { } }) - t.Run("ErrFrameFieldsAllowed", func(t *testing.T) { + t.Run("ErrFieldFieldsAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + if _, err := index.CreateField("f", pilosa.FieldOptions{ Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt}, }, @@ -136,7 +136,7 @@ func TestIndex_CreateFrame(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + if _, err := index.CreateField("f", pilosa.FieldOptions{ Fields: []*pilosa.Field{ {Name: "", Type: pilosa.FieldTypeInt}, }, @@ -149,7 +149,7 @@ func TestIndex_CreateFrame(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + if _, err := index.CreateField("f", pilosa.FieldOptions{ Fields: []*pilosa.Field{ {Name: "field0", Type: "bad_type"}, }, @@ -162,7 +162,7 @@ func TestIndex_CreateFrame(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + if _, err := index.CreateField("f", pilosa.FieldOptions{ Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, }, @@ -174,21 +174,21 @@ func TestIndex_CreateFrame(t *testing.T) { }) } -// Ensure index can delete a frame. -func TestIndex_DeleteFrame(t *testing.T) { +// Ensure index can delete a field. +func TestIndex_DeleteField(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame. + // Create field. if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - // Delete frame & verify it's gone. + // Delete field & verify it's gone. if err := index.DeleteField("f"); err != nil { t.Fatal(err) } else if index.Field("f") != nil { - t.Fatal("expected nil frame") + t.Fatal("expected nil field") } // Delete again to make sure it doesn't error. @@ -197,7 +197,7 @@ func TestIndex_DeleteFrame(t *testing.T) { } } -// Ensure index can delete a frame. +// Ensure index can delete a field. func TestIndex_InvalidName(t *testing.T) { path, err := ioutil.TempDir("", "pilosa-index-") if err != nil { diff --git a/pilosa.go b/pilosa.go index 2c0a8502d..6a7e74dfb 100644 --- a/pilosa.go +++ b/pilosa.go @@ -50,7 +50,7 @@ var ( ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") - ErrName = errors.New("invalid index or frame's name, must match [a-z0-9_-]") + ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]") ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") // ErrFragmentNotFound is returned when a fragment does not exist. @@ -78,7 +78,7 @@ type BadRequestError struct { error } -// Regular expression to validate index and frame names. +// Regular expression to validate index and field names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) // ColumnAttrSet represents a set of attributes for a vertical column in an index. diff --git a/pql/ast_test.go b/pql/ast_test.go index f5d75e9de..4b130c610 100644 --- a/pql/ast_test.go +++ b/pql/ast_test.go @@ -33,11 +33,11 @@ func TestCall_String(t *testing.T) { c := &pql.Call{ Name: "Range", Args: map[string]interface{}{ - "frame": "f", + "other": "f", "field0": &pql.Condition{Op: pql.GTE, Value: 10}, }, } - if s := c.String(); s != `Range(field0 >= 10, frame="f")` { + if s := c.String(); s != `Range(field0 >= 10, other="f")` { t.Fatalf("unexpected string: %s", s) } }) diff --git a/pql/parser_test.go b/pql/parser_test.go index 0e2a5c17d..411406815 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -135,7 +135,7 @@ func TestParser_Parse(t *testing.T) { // Parse with both child calls and arguments. t.Run("ChildrenAndArguments", func(t *testing.T) { - q, err := pql.ParseString(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`) + q, err := pql.ParseString(`TopN(Bitmap(id=100, field=other), field=f, n=3)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], @@ -143,9 +143,9 @@ func TestParser_Parse(t *testing.T) { Name: "TopN", Children: []*pql.Call{{ Name: "Bitmap", - Args: map[string]interface{}{"id": int64(100), "frame": "other"}, + Args: map[string]interface{}{"id": int64(100), "field": "other"}, }}, - Args: map[string]interface{}{"n": int64(3), "frame": "f"}, + Args: map[string]interface{}{"n": int64(3), "field": "f"}, }, ) { t.Fatalf("unexpected call: %#v", q.Calls[0]) @@ -154,14 +154,14 @@ func TestParser_Parse(t *testing.T) { // Parse a list argument. t.Run("ListArgument", func(t *testing.T) { - q, err := pql.ParseString(`TopN(frame="f", ids=[0,10,30])`) + q, err := pql.ParseString(`TopN(field="f", ids=[0,10,30])`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], &pql.Call{ Name: "TopN", Args: map[string]interface{}{ - "frame": "f", + "field": "f", "ids": []interface{}{int64(0), int64(10), int64(30)}, }, }, diff --git a/server/cluster_test.go b/server/cluster_test.go index 9327befc0..3dfc6c4d9 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -42,7 +42,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // Expected indexes and Frames + // Expected indexes and Fields expected := map[string][]string{ "i": []string{"f"}, } @@ -51,14 +51,14 @@ func TestMain_SendReceiveMessage(t *testing.T) { client0 := m0.Client() client1 := m1.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - // Make sure node0 knows about the index and frame created. + // Make sure node0 knows about the index and field created. schema0, err := client0.Schema(context.Background()) if err != nil { t.Fatal(err) @@ -66,15 +66,15 @@ func TestMain_SendReceiveMessage(t *testing.T) { received0 := map[string][]string{} for _, idx := range schema0 { received0[idx.Name] = []string{} - for _, frame := range idx.Fields { - received0[idx.Name] = append(received0[idx.Name], frame.Name) + for _, field := range idx.Fields { + received0[idx.Name] = append(received0[idx.Name], field.Name) } } if !reflect.DeepEqual(received0, expected) { t.Fatalf("unexpected schema on node0: %s", received0) } - // Make sure node1 knows about the index and frame created. + // Make sure node1 knows about the index and field created. schema1, err := client1.Schema(context.Background()) if err != nil { t.Fatal(err) @@ -82,8 +82,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { received1 := map[string][]string{} for _, idx := range schema1 { received1[idx.Name] = []string{} - for _, frame := range idx.Fields { - received1[idx.Name] = append(received1[idx.Name], frame.Name) + for _, field := range idx.Fields { + received1[idx.Name] = append(received1[idx.Name], field.Name) } } if !reflect.DeepEqual(received1, expected) { @@ -92,8 +92,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", col=1) - SetBit(row=1, frame="f", col=2400000) + SetBit(row=1, field="f", col=1) + SetBit(row=1, field="f", col=2400000) `); err != nil { t.Fatal(err) } @@ -206,7 +206,7 @@ func TestClusterResize_AddNode(t *testing.T) { // Create a client for each node. client0 := m0.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { @@ -250,7 +250,7 @@ func TestClusterResize_AddNode(t *testing.T) { client0 := m0.Client() //client1 := m1.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { @@ -259,8 +259,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", col=1) - SetBit(row=1, frame="f", col=1300000) + SetBit(row=1, field="f", col=1) + SetBit(row=1, field="f", col=1300000) `); err != nil { t.Fatal(err) } @@ -302,7 +302,7 @@ func TestClusterResize_AddNode(t *testing.T) { client0 := m0.Client() //client1 := m1.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { @@ -311,8 +311,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. Note that no data is placed on slice 1. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", col=1) - SetBit(row=1, frame="f", col=2400000) + SetBit(row=1, field="f", col=1) + SetBit(row=1, field="f", col=2400000) `); err != nil { t.Fatal(err) } @@ -455,7 +455,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { t.Run("ErrorRemoveWithoutReplicas", func(t *testing.T) { client0 := m0.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { @@ -466,7 +466,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // TODO: Deterministic node IDs would ensure consistent results setColumns := "" for i := 0; i < 20; i++ { - setColumns += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth) + setColumns += fmt.Sprintf("SetBit(row=1, field=\"f\", col=%d) ", i*pilosa.SliceWidth) } if _, err := m0.Query("i", "", setColumns); err != nil { diff --git a/server/server_test.go b/server/server_test.go index 495812cdf..4356ad241 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -54,17 +54,17 @@ func TestMain_Set_Quick(t *testing.T) { if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) } - if err := client.CreateField(context.Background(), "i", cmd.Frame, pilosa.FieldOptions{}); err != nil && err != pilosa.ErrFieldExists { + if err := client.CreateField(context.Background(), "i", cmd.Field, pilosa.FieldOptions{}); err != nil && err != pilosa.ErrFieldExists { t.Fatal(err) } - if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { + if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, field=%q, col=%d)`, cmd.ID, cmd.Field, cmd.ColumnID)); err != nil { t.Fatal(err) } } // Validate data. - for frame, frameSet := range SetCommands(cmds).Frames() { - for id, columnIDs := range frameSet { + for field, fieldSet := range SetCommands(cmds).Fields() { + for id, columnIDs := range fieldSet { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ @@ -73,7 +73,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -86,8 +86,8 @@ func TestMain_Set_Quick(t *testing.T) { } // Validate data after reopening. - for frame, frameSet := range SetCommands(cmds).Frames() { - for id, columnIDs := range frameSet { + for field, fieldSet := range SetCommands(cmds).Fields() { + for id, columnIDs := range fieldSet { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ @@ -96,7 +96,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -119,7 +119,7 @@ func TestMain_SetRowAttrs(t *testing.T) { m := test.MustRunMain() defer m.Close() - // Create frames. + // Create fields. client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) @@ -131,37 +131,37 @@ func TestMain_SetRowAttrs(t *testing.T) { t.Fatal(err) } - // Set columns on different rows in different frames. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { + // Set columns on different rows in different fields. + if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, field="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, field="z", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=3, field="neg", col=100)`); err != nil { t.Fatal(err) } // Set row attributes. - if _, err := m.Query("i", "", `SetRowAttrs(row=1, frame="x", x=100)`); err != nil { + if _, err := m.Query("i", "", `SetRowAttrs(row=1, field="x", x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="x", x=-200)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="x", x=-200)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="z", x=300)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="z", x=300)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=3, frame="neg", x=-0.44)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=3, field="neg", x=-0.44)`); err != nil { t.Fatal(err) } // Query row x/1. - if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=1, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -172,19 +172,19 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Query rows after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, field="neg")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -196,7 +196,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { m := test.MustRunMain() defer m.Close() - // Create frames. + // Create fields. client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) @@ -205,9 +205,9 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Set columns on row. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=101)`); err != nil { t.Fatal(err) } @@ -217,7 +217,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -228,7 +228,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) @@ -287,14 +287,14 @@ func TestMain_RecalculateHashes(t *testing.T) { t.Fatal("create index:", err) } if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{CacheType: "ranked"}); err != nil { - t.Fatal("create frame:", err) + t.Fatal("create field:", err) } // Set some columns data := []string{} for rowID := 1; rowID < 10; rowID++ { for columnID := 1; columnID < 100; columnID++ { - data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", col=%d)`, rowID, columnID)) + data = append(data, fmt.Sprintf(`SetBit(row=%d, field="f", col=%d)`, rowID, columnID)) } } if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { @@ -311,7 +311,7 @@ func TestMain_RecalculateHashes(t *testing.T) { // Run a TopN query on all nodes. The result should be the same as the target. for _, m := range cluster { - res, err := m.Query("i", "", `TopN(frame="f")`) + res, err := m.Query("i", "", `TopN(field="f")`) if err != nil { t.Fatal(err) } @@ -325,37 +325,37 @@ func TestMain_RecalculateHashes(t *testing.T) { // SetCommand represents a command to set a column. type SetCommand struct { ID uint64 - Frame string + Field string ColumnID uint64 } type SetCommands []SetCommand -// Frames returns the set of column ids for each frame/row. -func (a SetCommands) Frames() map[string]map[uint64][]uint64 { +// Fields returns the set of column ids for each field/row. +func (a SetCommands) Fields() map[string]map[uint64][]uint64 { // Create a set of unique commands. m := make(map[SetCommand]struct{}) for _, cmd := range a { m[cmd] = struct{}{} } - // Build unique ids for each frame & row. - frames := make(map[string]map[uint64][]uint64) + // Build unique ids for each field & row. + fields := make(map[string]map[uint64][]uint64) for cmd := range m { - if frames[cmd.Frame] == nil { - frames[cmd.Frame] = make(map[uint64][]uint64) + if fields[cmd.Field] == nil { + fields[cmd.Field] = make(map[uint64][]uint64) } - frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ColumnID) + fields[cmd.Field][cmd.ID] = append(fields[cmd.Field][cmd.ID], cmd.ColumnID) } // Sort each set of column ids. - for _, frame := range frames { - for id := range frame { - sort.Sort(uint64Slice(frame[id])) + for _, field := range fields { + for id := range field { + sort.Sort(uint64Slice(field[id])) } } - return frames + return fields } // GenerateSetCommands generates random SetCommand objects. @@ -364,7 +364,7 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { for i := range cmds { cmds[i] = SetCommand{ ID: uint64(rand.Intn(1000)), - Frame: "x", + Field: "x", ColumnID: uint64(rand.Intn(10)), } } diff --git a/server_test.go b/server_test.go index 5ea5653cc..6cbbe191e 100644 --- a/server_test.go +++ b/server_test.go @@ -21,7 +21,7 @@ func TestMonitorAntiEntropy(t *testing.T) { } err = client.CreateField(context.Background(), "balh", "fralh", pilosa.FieldOptions{}) if err != nil { - t.Fatalf("creating frame: %v", err) + t.Fatalf("creating field: %v", err) } time.Sleep(time.Millisecond * 2) diff --git a/stats_test.go b/stats_test.go index 806baa738..55e90a741 100644 --- a/stats_test.go +++ b/stats_test.go @@ -42,39 +42,39 @@ func TestMultiStatClient_Expvar(t *testing.T) { hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).ClearBit(0, 1) - if pilosa.Expvar.String() != `{"index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"}) - if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Gauge creates a unique key, subsequent Gauge calls will overwrite hldr.Stats.Gauge("g", 5, 1.0) hldr.Stats.Gauge("g", 8, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Set creates a unique key, subsequent sets will overwrite hldr.Stats.Set("s", "4", 1.0) hldr.Stats.Set("s", "7", 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Record timing duration and a uniquely Set key/value dur, _ := time.ParseDuration("123us") hldr.Stats.Timing("tt", dur, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Expvar histogram is implemented as a gauge hldr.Stats.Histogram("hh", 3, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } @@ -109,7 +109,7 @@ func TestStatsCount_TopN(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -138,7 +138,7 @@ func TestStatsCount_Bitmap(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, row=0)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(field=f, row=0)`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -155,12 +155,12 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { called := false e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - frame := e.Holder.Field("d", "f") - if frame == nil { - t.Fatal("frame not found") + field := e.Holder.Field("d", "f") + if field == nil { + t.Fatal("field not found") } - frame.Stats = &MockStats{ + field.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { if name != "SetRowAttrs" { t.Errorf("Expected SetRowAttrs, Results %s", name) @@ -168,7 +168,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -199,7 +199,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, field=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -257,7 +257,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { } } -func TestStatsCount_CreateFrame(t *testing.T) { +func TestStatsCount_CreateField(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -282,13 +282,13 @@ func TestStatsCount_CreateFrame(t *testing.T) { called = true }, } - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", nil)) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", nil)) if !called { t.Error("Count isn't called") } } -func TestStatsCount_DeleteFrame(t *testing.T) { +func TestStatsCount_DeleteField(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -313,7 +313,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { called = true }, } - http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/frame/f", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/field/f", strings.NewReader(""))) if !called { t.Error("Count isn't called") } diff --git a/test/cluster.go b/test/cluster.go index ee87d51a2..6b1efe665 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -114,19 +114,19 @@ func (t *TestCluster) CreateIndex(name string) error { return nil } -func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FieldOptions) error { +func (t *TestCluster) CreateField(index, field string, opt pilosa.FieldOptions) error { for _, c := range t.Clusters { idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{}) if err != nil { return err } - if _, err := idx.CreateField(frame, opt); err != nil { + if _, err := idx.CreateField(field, opt); err != nil { return err } } return nil } -func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { +func (t *TestCluster) SetBit(index, field, view string, rowID, colID uint64, x *time.Time) error { // Determine which node should receive the SetBit. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. slice := colID / pilosa.SliceWidth @@ -137,9 +137,9 @@ func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x * if c == nil { continue } - f := c.Holder.Field(index, frame) + f := c.Holder.Field(index, field) if f == nil { - return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + return fmt.Errorf("index/field does not exist: %s/%s", index, field) } _, err := f.SetBit(view, rowID, colID, x) if err != nil { diff --git a/test/fragment.go b/test/fragment.go index 2e97d6f5d..1280d3801 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -31,7 +31,7 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment with a temporary path. -func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fragment { +func NewFragment(index, field, view string, slice uint64, cacheType string) *Fragment { file, err := ioutil.TempFile("", "pilosa-fragment-") if err != nil { panic(err) @@ -39,7 +39,7 @@ func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fra file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice), + Fragment: pilosa.NewFragment(file.Name(), index, field, view, slice), RowAttrStore: MustOpenAttrStore(), } f.Fragment.CacheType = cacheType @@ -48,11 +48,11 @@ func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fra } // MustOpenFragment creates and opens an fragment at a temporary path. Panic on error. -func MustOpenFragment(index, frame, view string, slice uint64, cacheType string) *Fragment { +func MustOpenFragment(index, field, view string, slice uint64, cacheType string) *Fragment { if cacheType == "" { cacheType = pilosa.DefaultCacheType } - f := NewFragment(index, frame, view, slice, cacheType) + f := NewFragment(index, field, view, slice, cacheType) if err := f.Open(); err != nil { panic(err) diff --git a/test/holder.go b/test/holder.go index c0e8f1233..3cb0a8791 100644 --- a/test/holder.go +++ b/test/holder.go @@ -80,9 +80,9 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption return &Index{Index: idx} } -// MustCreateFrameIfNotExists returns a given frame. Panic on error. -func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Field { - f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(frame, pilosa.FieldOptions{}) +// MustCreateFieldIfNotExists returns a given field. Panic on error. +func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field { + f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(field, pilosa.FieldOptions{}) if err != nil { panic(err) } @@ -90,9 +90,9 @@ func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Field { } // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. -func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { +func (h *Holder) MustCreateFragmentIfNotExists(index, field, view string, slice uint64) *Fragment { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists(frame, pilosa.FieldOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) if err != nil { panic(err) } @@ -108,9 +108,9 @@ func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice } // MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. -func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { +func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, slice uint64) *Fragment { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists(frame, pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) if err != nil { panic(err) } diff --git a/time_test.go b/time_test.go index 233a2ae1a..3ab652455 100644 --- a/time_test.go +++ b/time_test.go @@ -65,7 +65,7 @@ func TestViewByTimeUnit(t *testing.T) { }) } -// Ensure all applicable frame names can be generated when mutating a time bit. +// Ensure all applicable field names can be generated when mutating a time bit. func TestViewsByTime(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) @@ -84,66 +84,66 @@ func TestViewsByTime(t *testing.T) { }) } -// Ensure sets of frames can be returned for a given time range. +// Ensure sets of fields can be returned for a given time range. func TestViewsByTimeRange(t *testing.T) { t.Run("Y", func(t *testing.T) { a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YM", func(t *testing.T) { a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YMD", func(t *testing.T) { a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YMDH", func(t *testing.T) { a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("M", func(t *testing.T) { a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("MD", func(t *testing.T) { a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("MDH", func(t *testing.T) { a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("D", func(t *testing.T) { a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("DH", func(t *testing.T) { a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("H", func(t *testing.T) { a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) } diff --git a/utils_test.go b/utils_test.go index 15053cd2b..6335d087f 100644 --- a/utils_test.go +++ b/utils_test.go @@ -104,20 +104,20 @@ func (t *ClusterCluster) CreateIndex(name string) error { return nil } -func (t *ClusterCluster) CreateFrame(index, frame string, opt FieldOptions) error { +func (t *ClusterCluster) CreateField(index, field string, opt FieldOptions) error { for _, c := range t.Clusters { idx, err := c.Holder.CreateIndexIfNotExists(index, IndexOptions{}) if err != nil { return err } - if _, err := idx.CreateField(frame, opt); err != nil { + if _, err := idx.CreateField(field, opt); err != nil { return err } } return nil } -func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { +func (t *ClusterCluster) SetBit(index, field, view string, rowID, colID uint64, x *time.Time) error { // Determine which node should receive the SetBit. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. slice := colID / SliceWidth @@ -128,9 +128,9 @@ func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, if c == nil { continue } - f := c.Holder.Field(index, frame) + f := c.Holder.Field(index, field) if f == nil { - return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + return fmt.Errorf("index/field does not exist: %s/%s", index, field) } _, err := f.SetBit(view, rowID, colID, x) if err != nil { diff --git a/view_test.go b/view_test.go index 5bef29ddd..217c4bbb4 100644 --- a/view_test.go +++ b/view_test.go @@ -30,14 +30,14 @@ type View struct { } // NewView returns a new instance of View with a temporary path. -func NewView(index, frame, name string) *View { +func NewView(index, field, name string) *View { path, err := ioutil.TempDir("", "pilosa-view-") if err != nil { panic(err) } v := &View{ - View: pilosa.NewView(path, index, frame, name, pilosa.DefaultCacheSize), + View: pilosa.NewView(path, index, field, name, pilosa.DefaultCacheSize), RowAttrStore: test.MustOpenAttrStore(), } v.View.RowAttrStore = v.RowAttrStore @@ -45,8 +45,8 @@ func NewView(index, frame, name string) *View { } // MustOpenView creates and opens an view at a temporary path. Panic on error. -func MustOpenView(index, frame, name string) *View { - v := NewView(index, frame, name) +func MustOpenView(index, field, name string) *View { + v := NewView(index, field, name) if err := v.Open(); err != nil { panic(err) } From 34cea8768360a25a8e6bf08dccd393b7a03c07b8 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 6 Jun 2018 11:44:16 -0500 Subject: [PATCH 51/78] Remove WebUI (now contained in a separate package) It now lives at https://github.com/pilosa/webui --- .travis.yml | 2 +- CONTRIBUTING.md | 8 +- Dockerfile | 2 +- Makefile | 16 +- docs/getting-started.md | 4 - docs/installation.md | 6 +- docs/webui.md | 22 +- filesystem.go | 42 -- handler.go | 23 +- handler_test.go | 29 -- server/server.go | 2 - statik/.gitignore | 1 - statik/filesystem.go | 37 -- webui/assets/chevron-down.png | Bin 3655 -> 0 bytes webui/assets/main.js | 594 ---------------------- webui/assets/nav-cluster-active.svg | 1 - webui/assets/nav-cluster.svg | 1 - webui/assets/nav-console-active.svg | 1 - webui/assets/nav-console.svg | 1 - webui/assets/nav-documentation-active.svg | 1 - webui/assets/nav-documentation.svg | 1 - webui/assets/nav_item1.svg | 1 - webui/assets/style.css | 336 ------------ webui/index.html | 123 ----- 24 files changed, 30 insertions(+), 1224 deletions(-) delete mode 100644 filesystem.go delete mode 100644 statik/.gitignore delete mode 100644 statik/filesystem.go delete mode 100644 webui/assets/chevron-down.png delete mode 100644 webui/assets/main.js delete mode 100644 webui/assets/nav-cluster-active.svg delete mode 100644 webui/assets/nav-cluster.svg delete mode 100644 webui/assets/nav-console-active.svg delete mode 100644 webui/assets/nav-console.svg delete mode 100644 webui/assets/nav-documentation-active.svg delete mode 100644 webui/assets/nav-documentation.svg delete mode 100644 webui/assets/nav_item1.svg delete mode 100644 webui/assets/style.css delete mode 100644 webui/index.html diff --git a/.travis.yml b/.travis.yml index 4d413ec3b..0e4f44d1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ env: - GOARCH=amd64 - GOARCH=amd64 ENTERPRISE=1 install: - - make install-dep install-statik vendor generate-statik + - make install-dep vendor script: - make test before_deploy: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c991caeab..2ad9ab13d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,7 +78,7 @@ Pilosa includes a Makefile that automates several tasks: make install ``` -- Install build dependencies (dep, statik, and protoc): +- Install build dependencies (dep and protoc): ```sh make install-build-deps @@ -114,12 +114,6 @@ Pilosa includes a Makefile that automates several tasks: make release ``` -- Generate static assets for the WebUI: - - ```sh - make generate-statik - ``` - - Regenerate protocol buffer files in `internal/`: ```sh diff --git a/Dockerfile b/Dockerfile index 7cdb93cc0..2b83c71c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM golang:1.10.2 as builder COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ - && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a" + && CGO_ENABLED=0 make install-dep install FLAGS="-a" FROM scratch diff --git a/Makefile b/Makefile index 3f17f0b2e..821ba33b6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc install install-build-deps install-dep install-protoc install-protoc-gen-gofast prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast test CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -88,16 +88,12 @@ install: vendor generate-protoc: require-protoc require-protoc-gen-gofast go generate github.com/pilosa/pilosa/internal -# `go generate` statik assets (WebUI) -generate-statik: require-statik - go generate github.com/pilosa/pilosa/statik - # `go generate` stringers generate-stringer: go generate github.com/pilosa/pilosa # `go generate` all needed packages -generate: generate-protoc generate-statik generate-stringer +generate: generate-protoc generate-stringer # Create Docker image from Dockerfile docker: @@ -126,23 +122,17 @@ endef require-dep: $(call require,dep) -require-statik: - $(call require,statik) - require-protoc-gen-gofast: $(call require,protoc-gen-gofast) require-protoc: $(call require,protoc) -install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc install-stringer +install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer install-dep: go get -u github.com/golang/dep/cmd/dep -install-statik: - go get -u github.com/rakyll/statik - install-stringer: go get -u golang.org/x/tools/cmd/stringer diff --git a/docs/getting-started.md b/docs/getting-started.md index c5383e81c..276b0bff9 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -112,10 +112,6 @@ Note that both the user IDs and the repository IDs were remapped to sequential i #### Make Some Queries -
-

Note the Pilosa server comes with a WebUI for constructing queries in a browser. In local development, it is available at localhost:10101.

-
- Which repositories did user 14 star: ``` request curl localhost:10101/index/repository/query \ diff --git a/docs/installation.md b/docs/installation.md index cf743721f..639293338 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -142,11 +142,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) git clone https://github.com/pilosa/pilosa.git ``` -3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): +3. Build the Pilosa repo: ``` cd $GOPATH/src/github.com/pilosa/pilosa make install-build-deps - make generate-statik make install ``` @@ -294,11 +293,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende git clone https://github.com/pilosa/pilosa.git ``` -3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): +3. Build the Pilosa repo: ``` cd $GOPATH/src/github.com/pilosa/pilosa make install-build-deps - make generate-statik make install ``` diff --git a/docs/webui.md b/docs/webui.md index 07acd2ab4..94377fbe8 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -9,13 +9,27 @@ nav = [ ## WebUI -The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101). +A web-based app called Pilosa WebUI is available in a separate package. This can be used for constructing queries and viewing the cluster status. -This can be used for constructing queries and viewing the cluster status. +### Installation + +Releases are [available on Github](https://github.com/pilosa/webui/releases) as well as on [Homebrew](https://brew.sh/) for Mac. + +Installing on a Mac with Homebrew is simple; just run: + +``` +brew install pilosa-webui +``` + +You may also build from source by checking out the [repo on Github](https://github.com/pilosa/webui) and running: + +``` +make install +``` ### Console -The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. +The Console view allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. Each query's result will be displayed in the Output section along with the query time. @@ -39,4 +53,4 @@ Frame creation also supports options like `timeQuantum`. When creating a new fra ### Cluster Admin -Use the [Cluster Admin tab](http://localhost:10101/#admin) to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames. +Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames. diff --git a/filesystem.go b/filesystem.go deleted file mode 100644 index 5664f0987..000000000 --- a/filesystem.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "fmt" - "net/http" -) - -// Ensure nopFileSystem implements interface. -var _ FileSystem = &nopFileSystem{} - -// FileSystem represents an interface for a WebUI file system. -type FileSystem interface { - New() (http.FileSystem, error) -} - -func init() { - NopFileSystem = &nopFileSystem{} -} - -// NopFileSystem represents a FileSystem that returns an error if called. -var NopFileSystem FileSystem - -type nopFileSystem struct{} - -// New is a no-op implementation of FileSystem New method. -func (n *nopFileSystem) New() (http.FileSystem, error) { - return nil, fmt.Errorf("file system not implemented") -} diff --git a/handler.go b/handler.go index 083d40279..c4bb364ea 100644 --- a/handler.go +++ b/handler.go @@ -42,8 +42,6 @@ import ( type Handler struct { Handler http.Handler - FileSystem FileSystem - Logger Logger // Keeps the query argument validators for each handler @@ -87,8 +85,7 @@ func OptHandlerAllowedOrigins(origins []string) HandlerOption { // NewHandler returns a new instance of Handler with a default logger. func NewHandler(opts ...HandlerOption) (*Handler, error) { handler := &Handler{ - FileSystem: NopFileSystem, - Logger: NopLogger, + Logger: NopLogger, } handler.Handler = NewRouter(handler) handler.populateValidators() @@ -137,8 +134,7 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler { // NewRouter creates a new mux http router. func NewRouter(handler *Handler) *mux.Router { router := mux.NewRouter() - router.HandleFunc("/", handler.handleWebUI).Methods("GET") - router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") + router.HandleFunc("/", handler.handleHome).Methods("GET") router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") @@ -226,19 +222,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { - // If user is using curl, don't chuck HTML at them - if strings.HasPrefix(r.UserAgent(), "curl") { - http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the WebUI by visiting this URL in your browser.", http.StatusNotFound) - return - } - filesystem, err := h.FileSystem.New() - if err != nil { - _ = h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - h.Logger.Printf("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") - return - } - http.FileServer(filesystem).ServeHTTP(w, r) +func (h *Handler) handleHome(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.", http.StatusNotFound) } // handleGetSchema handles GET /schema requests. diff --git a/handler_test.go b/handler_test.go index 036512de4..f9e490d9d 100644 --- a/handler_test.go +++ b/handler_test.go @@ -31,7 +31,6 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/statik" "github.com/pilosa/pilosa/test" ) @@ -915,34 +914,6 @@ func TestHandler_RecalculateCaches(t *testing.T) { } -func TestHandler_WebUI(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.FileSystem = &statik.FileSystem{} - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil)) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - if !strings.Contains(w.Body.String(), "Pilosa WebUI") { - t.Fatalf("WebUI is not being served correctly.") - } - - // If curl is the client, the response should be different - w = httptest.NewRecorder() - req := test.MustNewHTTPRequest("GET", "/", nil) - req.Header.Add("User-Agent", "curl/7.54.0") - h.ServeHTTP(w, req) - if !strings.Contains(w.Body.String(), "try the WebUI") { - t.Fatalf("WebUI is not being served correctly.") - } -} - func TestHandler_CORS(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() diff --git a/server/server.go b/server/server.go index 345807ec2..bb75a1d2e 100644 --- a/server/server.go +++ b/server/server.go @@ -39,7 +39,6 @@ import ( "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gopsutil" "github.com/pilosa/pilosa/gossip" - "github.com/pilosa/pilosa/statik" "github.com/pilosa/pilosa/statsd" "github.com/pkg/errors" ) @@ -170,7 +169,6 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "wrapping handler") } handler.Logger = m.logger - handler.FileSystem = &statik.FileSystem{} handler.API = pilosa.NewAPI() handler.API.Logger = m.logger diff --git a/statik/.gitignore b/statik/.gitignore deleted file mode 100644 index 485c0c57d..000000000 --- a/statik/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/statik.go diff --git a/statik/filesystem.go b/statik/filesystem.go deleted file mode 100644 index e3bf95cb1..000000000 --- a/statik/filesystem.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//go:generate statik -src=../webui -dest=.. -// -// Package statik contains static assets for the Web UI. `go generate` or -// `make generate-statik` will produce statik.go, which is ignored by git. -package statik - -import ( - "net/http" - - "github.com/pilosa/pilosa" - "github.com/rakyll/statik/fs" -) - -// Ensure nopFileSystem implements interface. -var _ pilosa.FileSystem = &FileSystem{} - -// FileSystem represents a static FileSystem. -type FileSystem struct{} - -// New is a statik implementation of FileSystem New method. -func (s *FileSystem) New() (http.FileSystem, error) { - return fs.New() -} diff --git a/webui/assets/chevron-down.png b/webui/assets/chevron-down.png deleted file mode 100644 index 3312489a21a768a90cb73633e3a596c792c6941b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3655 zcmc&%=Q|q?*KW0H)Tmae-BOjBMU9A24O;c6Q8k0w8WKdQtwyLI8k;CJ(x67|hSaE^ z6|2;a74t_Bd(?V7f5ZFXJ=b;4{pDQub)OICzRrm;eEE!(nV0#}rAw^3&$Wy%u=9V( zc3m3+?#C;WRNBP}*6USt8_amab zi^h;Is}+fy%21M@F?ele)cv1>K-vLyLy5L`@9Kq!|BqORPvUBg^Evj<4W(8q|CaQ; zfj-@C?_4rA8i1G*|KyrqU(f8R^Y;SmXTWXiW6t^`_&;3P^h6t$^@M78kW@#OmD7~* zC|u%g=y$tpBOr`mX;U|C2HN(p8G3T+?HtrD=b=rO-Q^KNT;89bpZvm4NJT6n*|)zE zH*h}*TEI5K>wi;9jDw5H`ChB_6Kz{gxvT1FGSkfq$ldWr2)fE&S+}zI7)X454~Nql zKY5qjkSG-Qtt929%76^%4{Wn64~11!O!B&4PgslhUG77Id-+HeIFwAv7n(TISXbMx zdVKnKav*z6;sq;e$qEU5_eW*2GbM5U>)3%OTO7~>FbY`KVRjM&&J<=Ip{(WQR7OzS z?|W9K_o~tI7#Y!JN%SdYiFKL3bY3+>;*Pi9>FD@MORsP5W_E??c2kPLG?+TJFhplI zs3Ovd&Ew8Bw{4RZiYAWt06Pf)E762Mzj~W05~~yi;1_~R9M7=HHMKgyl%fydz&lVe?g%mrA+|nx`3FA$lpXH!a5g6%^eoR~m?XViC z4kuyyw4Dt4d@jaKezLBTVBHH2D9vM(fk;$V=}2^-98zh-+{&!v*cUk%8frW58Cj{m z_iB!cJY5w{ZZ4h0J5JeDZh<9bxf)`k3KKoHsU3^}?O5A^6skLl4oyL~-9D0TH$P94 z6fx2H;PAAY=k zvSjJ}ONWkOA+!KVgcnX$=~PPnOFUO&e_7XBb|;KSp+xv}N2kaAAW6 zqQfx%l0s?Wh(zxs?__TUp+8r$BBKD_)^7UEj{Ovc&P}-O`lczKYw?~MH$u?`GPyLl z&wqu7Ej=qK8Qc9&BgkwgpqWLP*g|jl!zqOYKNK9?mmB&vhR4isJg;+#i5pjI^9v@e zcT$5&6$6MZyM(5_Xjzu~)-Y*0XX@{p*yCo3m<H?JL2_HGoe|6nCZiLU5)Ec4Gpk}C9k02A?A{RX8b=e!nZ z#p&nW0#MIwJUzUfH4KNFW6z97F`Pt-8Q^3G5Lk=A#1mUHodWc zv)!LJdttEI$%n`=t{h%CFb#dp!_lQ_N1Y1&2VXAX&ByH&9j;w7ble)N1-$zf1c?D3ir>m1$TR*V;wUV(u z*xID@%^BW(z7p(Ohe@vp3x9VjCH@%!wYBYE_DX&fLnZn29dAmtWg}+QNl463V3^q0 z344g3hma4UY&!~DaA#4@{!eXT*k4c)1kXKg_-J(TT^Zk~5sC?d0(NA3KbbW}Ca4t; z*T;>~wdDlZL%CKwz3F**!YoLM=D7%vZ@_Z^iKSYX80ke zVh{|ve`k@m<_}nAWnf>1Nr5*lD7mf0X`vF4A)3K4(^6589-=pk?)MT{$uwX+{*UEX0E%$Wkise(G1v78Nj%2ChfV>T2SxAbjQcF@)!q7}MZSjU&SPTk-mRJ8wD zyKZ3j?1;gFobG@kzXW zdh(K+ekHT~=%#|{O2sYPn!$*2y~-r_I^!#DB;uki7l9x*wgB5w^d~1mc8^rr79f=P z9l-64^Ry?1ObR*0V)bUvcjkW0|m5CH9Ip${Ay_Bd$j)lW-=F*({6B$h}(qWM36| z@s@e0V$otv9TPr!=2Sg3Mq4H{x~ZGmc@935%hz$Aurh_tD&q3qUdlH?O646w;e!&O zY}`Vk{dNyb%#^S`HtC+`HmKRh>vbj>ysWF6ty_~s9>gKb@s&u7OiYzU4E@h+|Y~E#o^~cyP&no8D67>AS zYv}r=rB`M|S>GiSwPI^a)12Vi) zg~7jSVPreXjW*8EZTFv*5i@i3c=PuxWmrL~-}5c}A;Cm8QlK4*woup+lu+1&4)_!m zzC1BxD4v&Pl;@A(@~EDV9(rbLb+Lmb+%+V&>_aylKReZQX%R&2XI_ZGRH6X=ByOU` z$CJls!#IOcBCamO88I^7nAuYsPSz~!f%sO?LO4G#CH)qL&B*-dYI@GsWO*08*`g(J zD=eUu!WvX*=*uQf<(IB%2It+2CR%={xc=P=$ss0%o*qDRAB+peGK*?$nf?d+wZjFH zrwyv@u5|Eg)wkO7JFxd~48Q|=-e)}F@B%bOMRu4S)7O1A*LO*__9sFd)5z%a_?&>e zzgWjmg)7N-8Z-XoR50ARt~Qanb8xnj0>i!~E4|`T2ggQveuz8NE|`K#gb;YM6064F zPmElfSi)7WONW_dXkOlAGO9H%Uke&6*}#4W-uhNgvz}-dpBQ=n7OR*kz`X9hZQ=2K zYt%U}aB>EvO`Tj|RoIp&Ht2?5|1!Ht1n28rpMGPyM}*?#1KTM~B7uI^)h|P&Uq>`g zUgNxcX6>0_q2RraA%0A5Z1_B?Nu49xZCPFhhzEHbR6xIH+ajvm*0_0{eT!PU*auaX zzC6q|2S)ymaDuP5S8Uwf{Y*V-p!4Q4q;p9`;8$%O@7J^co#Xr4!<)9MkO~H{t&UCNKlt diff --git a/webui/assets/main.js b/webui/assets/main.js deleted file mode 100644 index 265f51997..000000000 --- a/webui/assets/main.js +++ /dev/null @@ -1,594 +0,0 @@ -class REPL { - constructor(input, output, button, completer) { - this.input = input - this.output = output - this.button = button - this.completer = completer - this.history = [] - this.history_index = 0 - this.history_buffer = '' - this.result_number = 0 - } - bind_events() { - var repl = this - var keys = { - TAB: 9, - ENTER: 13, - UP_ARROW: 38, - DOWN_ARROW: 40 - } - - this.input.addEventListener("keydown", function(e) { - if (e.keyCode == keys.UP_ARROW) { - e.preventDefault() - if (repl.input.value.substring(0, repl.input.selectionStart).indexOf('\n') == '-1') { - if (repl.history_index == 0) { - return - } else { - if (repl.history_index == repl.history.length) { - repl.history_buffer = repl.input.value - } - repl.history_index-- - repl.input.value = repl.history[repl.history_index] - repl.input.setSelectionRange(repl.input.value.length, repl.input.value.length) - } - } - } - if (e.keyCode == keys.DOWN_ARROW) { - e.preventDefault() - if (repl.input.value.substring(repl.input.selectionEnd, repl.input.length).indexOf('\n') == '-1') { - if (repl.history_index == repl.history.length) { - return - } else { - repl.history_index++ - if (repl.history_index == repl.history.length) { - repl.input.value = repl.history_buffer - } else { - repl.input.value = repl.history[repl.history_index] - } - repl.input.setSelectionRange(repl.input.value.length, repl.input.value.length) - } - } - } - if (e.keyCode == keys.ENTER && !e.shiftKey) { - e.preventDefault() - repl.submit(); - } - if (e.keyCode == keys.TAB) { - e.preventDefault() - repl.completer.complete() - } - }) - repl.button.onclick = function() { - repl.submit(); - }; - } - - submit() { - this.history_buffer = '' - this.history_index = this.history.length - this.history[this.history_index] = this.input.value - this.history_index++ - this.process_query(this.input.value) - this.input.value = "" - } - - process_query(query) { - var xhr = new XMLHttpRequest(); - var url, data, request, command_name; - var e = document.getElementById("index-dropdown"); - var indexname = e.options[e.selectedIndex].text; - var repl = this; - if (query.startsWith(":")) { - var parsed_query = parse_query(query, indexname); - if (Object.keys(parsed_query).length === 0) { - repl.create_single_output({ - "input": query, - "output": "invalid query", - "status": 400, - "indexname": indexname, - }); - return; - } else { - // set selectedIndex from dropdown list - if (parsed_query.command === "use") { - for (var i = 0; i < e.options.length; i++) { - if (e.options[i].text === parsed_query.command_name) { - e.selectedIndex = i; - break; - } - } - return; - } - url = parsed_query.url; - data = parsed_query.data; - request = parsed_query.request; - command_name = parsed_query.command_name; - } - } - else { - url = '/index/' + indexname + '/query' - request = "POST" - data = query - } - xhr.open(request, url); - xhr.setRequestHeader('Content-Type', 'application/text'); - - var start_time = new Date().getTime(); - xhr.onload = function () { - var end_time = new Date().getTime(); - repl.result_number++ - repl.create_single_output({ - "input": query, - "output": xhr.responseText, - "status": xhr.status, - "indexname": indexname, - "querytime_ms": end_time - start_time, - }); - }; - - xhr.send(data); - // Remove index from dropdown with delete index command - if (request === 'DELETE' && url === '/index/' + command_name){ - for (var i = 0; i < e.options.length; i++) { - if (e.options[i].text === command_name) { - e.remove(i); - break; - } - } - } - } - - create_single_output(res) { - var node = document.createElement("div"); - node.classList.add('output'); - var output_string = res['output'] - var result_class = "result-output" - var getting_started_errors = [ - 'index not found', - 'frame not found', - ] - var output_json; - if (isJSON(output_string)) { - output_json = JSON.parse(output_string) - } - // handle output formatting - if (res["status"] != 200) { - result_class = "result-error"; - if (output_json) { - if ("error" in output_json) { - if (getting_started_errors.indexOf(output_json['error']) >= 0) { - output_string += `
-
- Just getting started? Try this:
- :create index test
- :use test
- :create frame foo
- SetBit(rowID=0, columnID=0, frame=foo) # Use PQL to set a bit - ` - } - } - } - } - - - var markup =` -
-
-
-
-
Input
-        - Source: ${res.indexname} -
-
- ${res.input} -
-
-
-
-
output
-        - ${res.querytime_ms} ms -
-
- ${output_string} -
-
Expand
- -
-
-
- -
-
- ` - node.innerHTML = markup; - this.output.insertBefore(node, this.output.firstChild); - - // Expand when overflow - var element = this.output.firstChild.getElementsByClassName(result_class)[0]; - var expand = this.output.firstChild.getElementsByClassName("expand")[0]; - if (element.clientHeight < element.scrollHeight) { - expand.style.display = 'block'; - } else { - expand.style.display = 'none'; - } - expand.onclick = function () { - element.style.height = element.scrollHeight + "px"; - expand.style.display = 'none'; - return false; - }; - } - - populate_index_dropdown() { - var xhr = new XMLHttpRequest(); - xhr.open('GET', '/schema') - var select = document.getElementById('index-dropdown') - - xhr.onload = function() { - var schema = JSON.parse(xhr.responseText) - for(var i=0; i 0) { - select.value = 1; - } - } - xhr.send(null) - } - -} - -function populate_version() { - var xhr = new XMLHttpRequest(); - xhr.open('GET', '/version') - var node = document.getElementById('server-version') - - xhr.onload = function() { - var version = JSON.parse(xhr.responseText)['version'] - var version_major_minor = /v?(\d+\.\d+).*/.exec(version)[1] - var doc_link = document.getElementById('nav-documentation') - doc_link.onclick = function() { - window.open('https://www.pilosa.com/docs/v' + version_major_minor + '/introduction/') - } - node.innerHTML = "Pilosa v" + version - } - xhr.send(null) -} - -function handle_nav_click(e) { - // e.id = "nav-xxx" - name = e.id.substring(4) - set_active_pane_by_name(name) - window.location.hash = name -} - -function set_active_pane_by_name(name) { - // toggle the nav buttons - document.getElementsByClassName("nav-active")[0].classList.remove("nav-active") - document.getElementById("nav-" + name).classList.add("nav-active") - - // toggle the main interface content divs - document.getElementsByClassName("interface-active")[0].classList.remove("interface-active") - document.getElementById('interface-' + name).classList.add("interface-active") - - // hack hack - switch(name) { - case "cluster": - update_cluster_status() - break - case "documentation": - open_external_docs() - break - } -} - - -function update_cluster_status() { - var xhr = new XMLHttpRequest(); - xhr.open('GET', '/status') - xhr.onload = function() { - var status = JSON.parse(xhr.responseText) - render_status(status) - } - xhr.send(null) - - var xhrSchema = new XMLHttpRequest(); - xhrSchema.open('GET', '/schema') - xhrSchema.onload = function() { - var schema = JSON.parse(xhrSchema.responseText) - render_schema(schema) - } - xhrSchema.send(null) -} - -function render_status(status) { - // render node table - var nodes_div = document.getElementById("status-nodes") - while (nodes_div.firstChild) { - nodes_div.removeChild(nodes_div.firstChild); - } - - var nodes = status["nodes"] - table = document.createElement("table") - tbody = document.createElement("tbody") - table.appendChild(tbody) - var caption = document.createElement("caption") - caption.innerHTML = "(" + nodes.length + ")" - table.appendChild(caption) - - var header = document.createElement('tr') - markup = `Host - ID - Coordinator` - header.innerHTML = markup - tbody.appendChild(header) - for(var n=0; n${nodes[n]["uri"]["host"]}:${nodes[n]["uri"]["port"]} - ${nodes[n]["id"]} - ${nodes[n]["isCoordinator"]}` - row.innerHTML = markup - tbody.appendChild(row) - } - nodes_div.appendChild(table) -} - -function render_schema(schema) { - // render index tables - var indexes_div = document.getElementById("status-indexes") - while (indexes_div.firstChild) { - indexes_div.removeChild(indexes_div.firstChild); - } - - var indexes = schema["indexes"] // TODO currently comes from only node 0 - for(var n=0; nName - Cache Type - Cache Size` - header.innerHTML = markup - tbody.appendChild(header) - - var frames = indexes[n]["frames"] - if(frames) { - for(var m=0; m${frames[m]["name"]} - ${frames[m]["options"]["cacheType"]} - ${frames[m]["options"]["cacheSize"]}` - tbody.appendChild(row) - } - } - indexes_div.appendChild(table) - } - - // render slice tables - // TODO enable when Slices element is present in status response - /* - var slices_div = document.getElementById("status-slices") - data = "" - for(var n=0; n" - } - } - slices_div.innerHTML = data - */ - -} - -function open_external_docs() { - window.open("https://www.pilosa.com/docs"); -} - -function check_anchor_uri() { - var pane_names = {"console": 0, "cluster": 0, "documentation": 0} - var anchor = window.location.hash.substr(1); - if(anchor in pane_names) { - set_active_pane_by_name(anchor) - } -} - -Date.prototype.today = function () { - return this.getFullYear() +"/"+ (((this.getMonth()+1) < 10)?"0":"") + (this.getMonth()+1) +"/"+ ((this.getDate() < 10)?"0":"") + this.getDate(); -} - -Date.prototype.timeNow = function () { - return ((this.getHours() < 10)?"0":"") + this.getHours() +":"+ ((this.getMinutes() < 10)?"0":"") + this.getMinutes() +":"+ ((this.getSeconds() < 10)?"0":"") + this.getSeconds(); -} - -populate_version() - - -class Autocompleter { - constructor(input, output) { - this.input = input - this.output = output - this.keyword_map = this.static_keywords - this.init_dynamic_keywords() - } - - get static_keywords() { - return { - // keyword: length of substring that comes after cursor - "SetBit()": 1, - "ClearBit()": 1, - "SetRowAttrs()": 1, - "SetColumnAttrs()": 1, - "Bitmap()": 1, - "Union()": 1, - "Intersect()": 1, - "Difference()": 1, - "Count()": 1, - "Range()": 1, - "TopN()": 1, - "frame=": 0, - } - } - - complete() { - var completer = this - // extract word fragment ending at cursor. a word fragment: - // - starts with last nonalpha character before cursor (or beginning of string) - // - ends at cursor - var word_start = completer.input.selectionEnd-1 - while(word_start>0) { - var c = completer.input.value.charCodeAt(word_start) - if(!((c>64 && c<91) || (c>96 && c<123))) { - word_start++ - break - } - word_start-- - } - var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd) - - // check for keyword match and insert if exactly one match - var matches = [] - for(var keyword in this.keyword_map) { - if(keyword.startsWith(input_word)){ - matches.push(keyword) - } - } - if(matches.length > 1) { - // completer.output.innerHTML = whatever - } - - if(matches.length == 1) { - // completer.output.innerHTML = "" - var cursor_pos = completer.input.selectionEnd - var completion = matches[0].substring(input_word.length) - var before = completer.input.value.substring(0, cursor_pos) - var after = completer.input.value.substring(cursor_pos) - completer.input.value = before + completion + after - var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]] - completer.input.setSelectionRange(new_pos, new_pos) - } - } - - init_dynamic_keywords() { - // hit /schema, parse indexes, frames, add to list - } - - add_keyword() { - // call when index or frame created in webui - } - - remove_keyword() { - // call when index or frame deleted in webui - // issue: if e.g. multiple indexes have same frame, removing one removes all. - // solution: maintain count. requires more elaborate representation of keywords. - } -} - -var input = document.getElementById('query') -var output = document.getElementById('outputs') -var button = document.getElementById('query-btn') -var autocomplete_output = document.getElementById('autocomplete-container') - -autocompleter = new Autocompleter(input, autocomplete_output) -repl = new REPL(input, output, button, autocompleter) -repl.populate_index_dropdown() -repl.bind_events() - -input.focus() - -check_anchor_uri() - -function isJSON(str) { - try { - JSON.parse(str) - } catch (e) { - return false - } - return true -} - -function parse_query(query, indexname) { - var keys = query.replace(/\s+/g, " ").split(" "); - var command = keys[0]; - var command_type = keys[1]; - var command_name = keys[2]; - var option_str = keys.slice(3, keys.length) - var options = parse_options(option_str); - if (command !== ":use") { - if (!command_name){ - return {} - } - } - - var parsed_query = {}; - parsed_query["command"] = command.substr(1, command.length); - parsed_query["command_name"] = command_name; - switch (command) { - case ":create": - parsed_query["request"] = "POST"; - if(Object.keys(options).length === 0) { - parsed_query["data"] = ""; - } else { - var opts = {"options":{}}; - for (var o in options) { - opts.options[o] = options[o] - } - parsed_query["data"] = JSON.stringify(opts); - } - switch (command_type){ - case "index": - parsed_query["url"] = '/index/' + command_name; - break; - case "frame": - parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name; - break - } - break; - case ":delete": - parsed_query["request"] = "DELETE"; - switch (command_type){ - case "index": - parsed_query["url"] = '/index/' + command_name; - parsed_query["data"] = ""; - break; - case "frame": - parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name; - parsed_query["data"] = ""; - break; - } - break; - case ":use": - parsed_query["command_name"] = keys[1]; - break; - default: - return {} - } - return parsed_query; -} - -function parse_options(option_str) { - var int_keys = ["cacheSize"]; - var options = {}; - for (var i = 0; i < option_str.length; i++) { - var parts = option_str[i].split('='); - if (int_keys.indexOf(parts[0]) !== -1 ){ - options[parts[0]] = Number(parts[1]) - } else { - options[parts[0]] = parts[1] - } - } - return options; -} diff --git a/webui/assets/nav-cluster-active.svg b/webui/assets/nav-cluster-active.svg deleted file mode 100644 index 871d6f2af..000000000 --- a/webui/assets/nav-cluster-active.svg +++ /dev/null @@ -1 +0,0 @@ -nav_cluster_1 diff --git a/webui/assets/nav-cluster.svg b/webui/assets/nav-cluster.svg deleted file mode 100644 index baf6310ea..000000000 --- a/webui/assets/nav-cluster.svg +++ /dev/null @@ -1 +0,0 @@ -nav_cluster_1 \ No newline at end of file diff --git a/webui/assets/nav-console-active.svg b/webui/assets/nav-console-active.svg deleted file mode 100644 index 2263c2a69..000000000 --- a/webui/assets/nav-console-active.svg +++ /dev/null @@ -1 +0,0 @@ -nav_console \ No newline at end of file diff --git a/webui/assets/nav-console.svg b/webui/assets/nav-console.svg deleted file mode 100644 index e9d2a9569..000000000 --- a/webui/assets/nav-console.svg +++ /dev/null @@ -1 +0,0 @@ -nav_console diff --git a/webui/assets/nav-documentation-active.svg b/webui/assets/nav-documentation-active.svg deleted file mode 100644 index b6e994980..000000000 --- a/webui/assets/nav-documentation-active.svg +++ /dev/null @@ -1 +0,0 @@ -documentation diff --git a/webui/assets/nav-documentation.svg b/webui/assets/nav-documentation.svg deleted file mode 100644 index 12cd2108c..000000000 --- a/webui/assets/nav-documentation.svg +++ /dev/null @@ -1 +0,0 @@ -documentation \ No newline at end of file diff --git a/webui/assets/nav_item1.svg b/webui/assets/nav_item1.svg deleted file mode 100644 index d6d7f779e..000000000 --- a/webui/assets/nav_item1.svg +++ /dev/null @@ -1 +0,0 @@ -nav_item1 \ No newline at end of file diff --git a/webui/assets/style.css b/webui/assets/style.css deleted file mode 100644 index 2cf65d086..000000000 --- a/webui/assets/style.css +++ /dev/null @@ -1,336 +0,0 @@ -*{ - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; -} -body{ - font-family: sans-serif; - background-color: #fbfcfd; - margin: 0; - color: #102445; -} -h2{ - margin-bottom: 30px; -} - -h5{ - text-transform: uppercase; - letter-spacing: 2px; - line-height: 1.21; - margin: 0; -} -a{ - line-height: 1.38; - letter-spacing: 0.2px; - text-decoration: none; - color: #102445; -} - - -a:hover{ - color: #1db598; -} - -textarea{ - width: 100%; - margin-bottom: 10px; - border-radius: 2px; - background-color: #fbfcfd; - border: solid 1.5px #e4eff4; - font-family: monospace; - font-size: 16px; - line-height: 1.5; - letter-spacing: 1.1px; - outline: none; - padding: 30px; -} - - -select{ - /*-webkit-appearance: none; - -moz-appearance: none; - appearance: none; - background: url("img/chevron-down.png") no-repeat calc(100% - 10px) !important;*/ - border-radius: 3px; - background-color: #fbfcfd; - width: 187px; - height: 50px; - border: solid 1.5px #e4eff4; - font-size: 18px; - font-weight: bold; - line-height: 1.39; - letter-spacing: 0.2px; - color: #102445; - padding: 10.5px; - -} - -button{ - width: 165px; - height: 50px; - border-radius: 3px; - background-color: #1db598; - outline: none; - border: none; - font-size: 16px; - color: white; -} - -em{ - font-style: normal; - opacity: 0.5; - font-size: 14px; - font-weight: 500; - letter-spacing: 0.2px; - color: #102445; -} - -.header{ - height: 92px; - display: flex; - align-items: center; - justify-content: space-between; - width: 90%; - margin: auto; -} - -.container{ - display: flex; - height:100%; - min-height: 100vh; -} -.nav{ - color: white; - display: flex; - flex-direction: column; - width: 150px; - background: #3c5f8d; -} - -.nav-item{ - height:150px; - display: flex; - align-items: center; - justify-content: center; - flex-direction: column; - border-bottom: 3px solid #2a4871; - cursor: pointer; -} - -.nav-active{ - background: #f2f7f9; - font-weight: bold; - color: #1db598; -} - -.nav-item > .nav-image { - display: flex; -} - -.nav-item > .nav-image-active { - display: none; -} - -.nav-active > .nav-image { - display: none; -} - -.nav-active > .nav-image-active { - display: flex; -} - - -.interface{ - display: none; - flex: 1; - flex-direction: column; - align-items: center; - background: #f2f7f9; -} - -.interface-active{ - display: flex; -} - -.query{ - margin-bottom: 30px; -} -.query, -.output-container, -.status-container{ - width: 75%; -} - -.output{ - margin-bottom: 30px; -} - -.input-controls{ - display: flex; - justify-content: flex-end; -} - -.tabs{ - display: flex; - background: #eaf2f6; -} -.active-tab{ - background: white; - font-weight: bold; - color: #1db598; - -} - -.tab{ - height:60px; - width: 100px; - border-top-right-radius: 5px; - display: flex; - align-items: center; - justify-content: center; - visibility: visible; - cursor: pointer; - -} - -.pane{ - background: white; - padding: 30px; - display: none; -} - -.active{ - display: block; -} - -.result-io-header{ - display: flex; - align-items: center; - margin-bottom: 15px; -} - -.result-input, -.result-output, -.result-error{ - height: 60px; - border-radius: 2px; - background-color: #fafafa; - border: solid 1.5px #e4eff4; - font-family: monospace; - font-size: 16px; - line-height: 1.5; - letter-spacing: 1.1px; - color: #102445; - padding: 15px; - margin-bottom: 15px; - word-break: break-all; - overflow-wrap: break-word; - overflow:hidden; -} - - -.result-output{ - background-color: #edf9f7; - border-left: solid 4px #1db598; -} - -.result-error{ - background-color: #fbf1f0; - border-left: solid 4px #fa3035; - color: #fa3035; -} - -.raw{ - height: 253px; - display: flex; - align-items: center; - justify-content: center; -} - - -.result-table > table { - border-left: solid 4px #1db598; -} - -table{ - border: solid 0.5px #e0e0e0; - width: 100%; - margin-bottom: 30px; - /*color:#3c5f8d;*/ -} -caption{ - text-align:left; - font-size: 16px; - font-weight: bold; - line-height: 1.21; - letter-spacing: 2px; - text-align: left; -} -th{ - font-size: 14px; - font-weight: bold; - line-height: 1.21; - letter-spacing: 2px; - color: #102445; - text-transform: uppercase; - text-align: left; - padding: 21px 30px; - background-color: white; -} -tr{ - border: solid 0.5px #e0e0e0; - background-color: white; -} -tr:nth-child(even) { - background-color: #f2f7f9; -} -td{ - padding: 21px 30px; -} - -.expand { - text-align: center; -} - -.query h2 { - display: inline-block; -} - -.query-tooltip { - position: relative; - display: inline; - color: #000; - margin-left: 5px; -} - -.query-tooltip:hover { - color: #000; -} - -.query-tooltip-content { - background-color: rgb(250, 250, 250); - border: solid 1.5px #e4eff4; - color: #102445; - border-radius: 2px; - padding: 15px; - margin-bottom: 15px; - - position: absolute; - left: 80px; - top: -30px; - z-index: 1; -} - -.query-tooltip-container { - position: relative; - visibility: hidden; -} - -.query-tooltip:hover+.query-tooltip-container{ - visibility: visible; -} - -.code{ - font-family: monospace; -} - diff --git a/webui/index.html b/webui/index.html deleted file mode 100644 index f6bc369de..000000000 --- a/webui/index.html +++ /dev/null @@ -1,123 +0,0 @@ - - - - - - - - Pilosa WebUI - - - -
- -
-
-
- -
- -
-

Query

- ? -
-
-
PQL
-
- SetBit(frame=foo, row=0, col=0)
- ClearBit(frame=foo, row=0, col=0)
- SetRowAttrs(frame=foo, row=0, color="blue")
- SetColumnAttrs(frame=foo, col=0, shape="circle")
- SetFieldValue(frame=foo, col=0, age=30)
- Bitmap(frame=foo, row=0)
- Range(frame=foo, row=0, start="2010-01", end="2017-03")
- Count(<BITMAP_CALL>)
- TopN([BITMAP_CALL], frame=foo, n=20)
- Union([BITMAP_CALL, ...])
- Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
- Difference(<BITMAP_CALL>, <BITMAP_CALL>)
- Xor([BITMAP_CALL, ...])
- Min([BITMAP_CALL], frame=foo, field=age)
- Max([BITMAP_CALL], frame=foo, field=age)
- Sum([BITMAP_CALL], frame=foo, field=age) -
-
-
Special commands
-
- :create index test
- :use test
- :create frame foo
- :delete index test
- :delete frame foo -
-
- <tab>: autocomplete
- <up>/<down>: history
-
-
- -
-
- -     - -
-
-
- -
-

Output

-
- -
-
- -
- -
-
-

Nodes

-
-
-
-
-

Indexes

-
-
-
-
- -
- -
- -
- docs! -
- -
- - - From 6f4c50a4b564529568b60ffb84a4fd64fdeb0920 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 6 Jun 2018 15:49:01 -0500 Subject: [PATCH 52/78] move fragment_test into the pilosa package (internal) --- attr.go | 53 +++ fragment_test.go => fragment_internal_test.go | 313 ++++++++++-------- index_internal_test.go | 46 +++ index_test.go | 3 + test/fragment.go | 103 ------ 5 files changed, 277 insertions(+), 241 deletions(-) rename fragment_test.go => fragment_internal_test.go (79%) create mode 100644 index_internal_test.go diff --git a/attr.go b/attr.go index 03ea4f43f..168ac0052 100644 --- a/attr.go +++ b/attr.go @@ -221,3 +221,56 @@ func DecodeAttrs(v []byte) (map[string]interface{}, error) { } return decodeAttrs(pb.GetAttrs()), nil } + +func newMemAttrStore() AttrStore { + return &memAttrStore{ + store: make(map[uint64]map[string]interface{}), + } +} + +// memAttrStore represents an in-memory implementation of the AttrStore interface. +type memAttrStore struct { + store map[uint64]map[string]interface{} +} + +// Path is an in-memory implementation of AttrStore Path method. +func (s *memAttrStore) Path() string { return "" } + +// Open is an in-memory implementation of AttrStore Open method. +func (s *memAttrStore) Open() error { + return nil +} + +// Close is an in-memory implementation of AttrStore Close method. +func (s *memAttrStore) Close() error { + return nil +} + +// Attrs returns a set of attributes by ID. +func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { + return s.store[id], nil +} + +// SetAttrs sets attribute values for a given ID. +func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { + s.store[id] = m + return nil +} + +// SetBulkAttrs sets attribute values for a set of ids. +func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { + for id, v := range m { + s.store[id] = v + } + return nil +} + +// Blocks is an in-memory implementation of AttrStore Blocks method. +func (s *memAttrStore) Blocks() ([]AttrBlock, error) { + return nil, nil +} + +// BlockData is an in-memory implementation of AttrStore BlockData method. +func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { + return nil, nil +} diff --git a/fragment_test.go b/fragment_internal_test.go similarity index 79% rename from fragment_test.go rename to fragment_internal_test.go index 747602348..20f3f7b6a 100644 --- a/fragment_test.go +++ b/fragment_internal_test.go @@ -12,20 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package pilosa import ( "bytes" "flag" + "io/ioutil" "math" "reflect" "testing" "testing/quick" "github.com/davecgh/go-spew/spew" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/test" ) // Test flags @@ -35,12 +34,9 @@ var ( FragmentPath = flag.String("fragment", "testdata/sample_view/0", "fragment path") ) -// SliceWidth is a helper reference to use when testing. -const SliceWidth = pilosa.SliceWidth - // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set bits on the fragment. @@ -60,7 +56,7 @@ func TestFragment_SetBit(t *testing.T) { } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) } else if n := f.Row(120).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) @@ -71,7 +67,7 @@ func TestFragment_SetBit(t *testing.T) { // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set and then clear bits on the fragment. @@ -89,7 +85,7 @@ func TestFragment_ClearBit(t *testing.T) { } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) } else if n := f.Row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) @@ -99,7 +95,7 @@ func TestFragment_ClearBit(t *testing.T) { // Ensure a fragment can set & read a value. func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set value. @@ -127,7 +123,7 @@ func TestFragment_SetValue(t *testing.T) { }) t.Run("Overwrite", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set value. @@ -155,7 +151,7 @@ func TestFragment_SetValue(t *testing.T) { }) t.Run("NotExists", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set value. @@ -185,7 +181,7 @@ func TestFragment_SetValue(t *testing.T) { values[i] = values[i] % (1 << bitDepth) } - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. @@ -223,7 +219,7 @@ func TestFragment_SetValue(t *testing.T) { func TestFragment_Sum(t *testing.T) { const bitDepth = 16 - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. @@ -248,7 +244,7 @@ func TestFragment_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if sum, n, err := f.Sum(pilosa.NewRow(2000, 4000, 5000), bitDepth); err != nil { + if sum, n, err := f.Sum(NewRow(2000, 4000, 5000), bitDepth); err != nil { t.Fatal(err) } else if n != 2 { t.Fatalf("unexpected count: %d", n) @@ -262,7 +258,7 @@ func TestFragment_Sum(t *testing.T) { func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. @@ -284,16 +280,16 @@ func TestFragment_MinMax(t *testing.T) { t.Run("Min", func(t *testing.T) { tests := []struct { - filter *pilosa.Row + filter *Row exp uint64 cnt uint64 }{ {filter: nil, exp: 0, cnt: 1}, - {filter: pilosa.NewRow(2000, 4000, 5000), exp: 300, cnt: 2}, - {filter: pilosa.NewRow(2000, 4000), exp: 300, cnt: 2}, - {filter: pilosa.NewRow(1), exp: 0, cnt: 0}, - {filter: pilosa.NewRow(1000), exp: 382, cnt: 1}, - {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, + {filter: NewRow(2000, 4000, 5000), exp: 300, cnt: 2}, + {filter: NewRow(2000, 4000), exp: 300, cnt: 2}, + {filter: NewRow(1), exp: 0, cnt: 0}, + {filter: NewRow(1000), exp: 382, cnt: 1}, + {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { if min, cnt, err := f.Min(test.filter, bitDepth); err != nil { @@ -308,16 +304,16 @@ func TestFragment_MinMax(t *testing.T) { t.Run("Max", func(t *testing.T) { tests := []struct { - filter *pilosa.Row + filter *Row exp uint64 cnt uint64 }{ {filter: nil, exp: 2818, cnt: 2}, - {filter: pilosa.NewRow(2000, 4000, 5000), exp: 2818, cnt: 1}, - {filter: pilosa.NewRow(2000, 4000), exp: 300, cnt: 2}, - {filter: pilosa.NewRow(1), exp: 0, cnt: 0}, - {filter: pilosa.NewRow(1000), exp: 382, cnt: 1}, - {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, + {filter: NewRow(2000, 4000, 5000), exp: 2818, cnt: 1}, + {filter: NewRow(2000, 4000), exp: 300, cnt: 2}, + {filter: NewRow(1), exp: 0, cnt: 0}, + {filter: NewRow(1000), exp: 382, cnt: 1}, + {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { if max, cnt, err := f.Max(test.filter, bitDepth); err != nil { @@ -336,7 +332,7 @@ func TestFragment_Range(t *testing.T) { const bitDepth = 16 t.Run("EQ", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. @@ -359,7 +355,7 @@ func TestFragment_Range(t *testing.T) { }) t.Run("NEQ", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. @@ -382,7 +378,7 @@ func TestFragment_Range(t *testing.T) { }) t.Run("LT", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. @@ -430,7 +426,7 @@ func TestFragment_Range(t *testing.T) { }) t.Run("GT", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. @@ -478,7 +474,7 @@ func TestFragment_Range(t *testing.T) { }) t.Run("BETWEEN", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. @@ -528,7 +524,7 @@ func TestFragment_Range(t *testing.T) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set and then clear bits on the fragment. @@ -548,7 +544,7 @@ func TestFragment_Snapshot(t *testing.T) { } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) } else if n := f.Row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) @@ -557,7 +553,7 @@ func TestFragment_Snapshot(t *testing.T) { // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set bits on the fragment. @@ -586,42 +582,42 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 200) - f.MustSetBits(101, 1) - f.MustSetBits(102, 1, 2) + f.mustSetBits(100, 1, 3, 200) + f.mustSetBits(101, 1) + f.mustSetBits(102, 1, 2) f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 2}); err != nil { + if pairs, err := f.Top(TopOptions{N: 2}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 100, Count: 3}) { + } else if pairs[0] != (Pair{ID: 100, Count: 3}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 102, Count: 2}) { + } else if pairs[1] != (Pair{ID: 102, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) } } // Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 200) - f.MustSetBits(101, 1) - f.MustSetBits(102, 1, 2) + f.mustSetBits(100, 1, 3, 200) + f.mustSetBits(101, 1) + f.mustSetBits(102, 1, 2) f.RecalculateCache() // Assign attributes. - f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)}) - f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)}) + f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": int64(10)}) + f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": int64(20)}) // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{ + if pairs, err := f.Top(TopOptions{ N: 2, FilterName: "x", FilterValues: []interface{}{int64(10), int64(15), int64(20)}, @@ -629,32 +625,32 @@ func TestFragment_Top_Filter(t *testing.T) { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 102, Count: 2}) { + } else if pairs[0] != (Pair{ID: 102, Count: 2}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 101, Count: 1}) { + } else if pairs[1] != (Pair{ID: 101, Count: 1}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) } } // Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() // Create an intersecting input row. - src := pilosa.NewRow(1, 2, 3) + src := NewRow(1, 2, 3) // Set bits on various rows. - f.MustSetBits(100, 1, 10, 11, 12) // one intersection - f.MustSetBits(101, 1, 2, 3, 4) // three intersections - f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections - f.MustSetBits(103, 1000, 1001, 1002) // no intersection + f.mustSetBits(100, 1, 10, 11, 12) // one intersection + f.mustSetBits(101, 1, 2, 3, 4) // three intersections + f.mustSetBits(102, 1, 2, 4, 5, 6) // two intersections + f.mustSetBits(103, 1000, 1001, 1002) // no intersection f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 3, Src: src}); err != nil { + if pairs, err := f.Top(TopOptions{N: 3, Src: src}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 3}, {ID: 102, Count: 2}, {ID: 100, Count: 1}, @@ -669,11 +665,11 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { t.Skip("short mode") } - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() // Create an intersecting input row. - src := pilosa.NewRow( + src := NewRow( 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, ) @@ -681,15 +677,15 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Set bits on rows 0 - 999. Higher rows have higher bit counts. for i := uint64(0); i < 1000; i++ { for j := uint64(0); j < i; j++ { - f.MustSetBits(i, j) + f.mustSetBits(i, j) } } f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 10, Src: src}); err != nil { + if pairs, err := f.Top(TopOptions{N: 10, Src: src}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 999, Count: 19}, {ID: 998, Count: 18}, {ID: 997, Count: 17}, @@ -707,18 +703,18 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) + f.mustSetBits(100, 1, 2, 3) + f.mustSetBits(101, 4, 5, 6, 7) + f.mustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.Top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 4}, {ID: 100, Count: 3}, }) { @@ -728,18 +724,18 @@ func TestFragment_TopN_IDs(t *testing.T) { // Ensure a fragment return none if CacheTypeNone is set func TestFragment_TopN_NopCache(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeNone) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeNone) defer f.Close() // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) + f.mustSetBits(100, 1, 2, 3) + f.mustSetBits(101, 4, 5, 6, 7) + f.mustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.Top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{}) { + } else if !reflect.DeepEqual(pairs, []Pair{}) { t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) } } @@ -750,17 +746,17 @@ func TestFragment_TopN_CacheSize(t *testing.T) { cacheSize := uint32(3) // Create Index. - index := test.MustOpenIndex() + index := mustOpenIndex() defer index.Close() // Create field. - field, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) + field, err := index.CreateFieldIfNotExists("f", FieldOptions{CacheType: CacheTypeRanked, CacheSize: cacheSize}) if err != nil { t.Fatal(err) } // Create view. - view, err := field.CreateViewIfNotExists(pilosa.ViewStandard) + view, err := field.CreateViewIfNotExists(ViewStandard) if err != nil { t.Fatal(err) } @@ -773,38 +769,34 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Close the storage so we can re-open it without encountering a flock. frag.Close() - f := &test.Fragment{ - Fragment: frag, - RowAttrStore: test.MustOpenAttrStore(), - } - f.Fragment.RowAttrStore = f.RowAttrStore + f := frag if err := f.Open(); err != nil { panic(err) } defer f.Close() // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) - f.MustSetBits(103, 8, 9, 10, 11, 12, 13) - f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14) - f.MustSetBits(105, 10, 11) + f.mustSetBits(100, 1, 2, 3) + f.mustSetBits(101, 4, 5, 6, 7) + f.mustSetBits(102, 8, 9, 10, 11, 12) + f.mustSetBits(103, 8, 9, 10, 11, 12, 13) + f.mustSetBits(104, 8, 9, 10, 11, 12, 13, 14) + f.mustSetBits(105, 10, 11) f.RecalculateCache() - p := []pilosa.Pair{ + p := []Pair{ {ID: 104, Count: 7}, {ID: 103, Count: 6}, {ID: 102, Count: 5}, } // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { + if pairs, err := f.Top(TopOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) > int(cacheSize) { t.Fatalf("TopN count cannot exceed cache size: %d", cacheSize) - } else if pairs[0] != (pilosa.Pair{ID: 104, Count: 7}) { + } else if pairs[0] != (Pair{ID: 104, Count: 7}) { t.Fatalf("unexpected pair(0): %v", pairs) } else if !reflect.DeepEqual(pairs, p) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(pairs)) @@ -813,14 +805,14 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Retrieve checksum and set bits. orig := f.Checksum() if _, err := f.SetBit(1, 200); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(pilosa.HashBlockSize*2, 200); err != nil { + } else if _, err := f.SetBit(HashBlockSize*2, 200); err != nil { t.Fatal(err) } @@ -832,11 +824,11 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Retrieve initial checksum. - var prev []pilosa.FragmentBlock + var prev []FragmentBlock // Set first bit. if _, err := f.SetBit(0, 0); err != nil { @@ -870,7 +862,7 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set bits on a different block. @@ -888,7 +880,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeLRU) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeLRU) defer f.Close() // Set bits on the fragment. @@ -899,19 +891,19 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { } // Verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.LRUCache); !ok { + if cache, ok := f.Cache().(*LRUCache); !ok { t.Fatalf("unexpected cache: %T", f.Cache()) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } // Reopen the fragment. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) } // Re-verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.LRUCache); !ok { + if cache, ok := f.Cache().(*LRUCache); !ok { t.Fatalf("unexpected cache: %T", f.Cache()) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) @@ -920,17 +912,17 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_RankCache_Persistence(t *testing.T) { - index := test.MustOpenIndex() + index := mustOpenIndex() defer index.Close() // Create field. - field, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) + field, err := index.CreateFieldIfNotExists("f", FieldOptions{CacheType: CacheTypeRanked}) if err != nil { t.Fatal(err) } // Create view. - view, err := field.CreateViewIfNotExists(pilosa.ViewStandard) + view, err := field.CreateViewIfNotExists(ViewStandard) if err != nil { t.Fatal(err) } @@ -949,22 +941,22 @@ func TestFragment_RankCache_Persistence(t *testing.T) { } // Verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.RankCache); !ok { + if cache, ok := f.Cache().(*RankCache); !ok { t.Fatalf("unexpected cache: %T", f.Cache()) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } // Reopen the index. - if err := index.Reopen(); err != nil { + if err := index.reopen(); err != nil { t.Fatal(err) } // Re-fetch fragment. - f = index.Field("f").View(pilosa.ViewStandard).Fragment(0) + f = index.Field("f").View(ViewStandard).Fragment(0) // Re-verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.RankCache); !ok { + if cache, ok := f.Cache().(*RankCache); !ok { t.Fatalf("unexpected cache: %T", f.Cache()) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) @@ -973,7 +965,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { - f0 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f0 := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f0.Close() // Set and then clear bits on the fragment. @@ -998,7 +990,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Read into another fragment. - f1 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f1 := mustOpenFragment("i", "f", ViewStandard, 0, "") if rn, err := f1.ReadFrom(&buf); err != nil { t.Fatal(err) } else if wn != rn { @@ -1016,7 +1008,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Close and reopen the fragment & verify the data. - if err := f1.Reopen(); err != nil { + if err := f1.reopen(); err != nil { t.Fatal(err) } else if n := f1.Cache().Len(); n != 1 { t.Fatalf("unexpected cache size (reopen): %d", n) @@ -1031,7 +1023,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) + f := NewFragment(*FragmentPath, "i", "f", ViewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -1047,7 +1039,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } func BenchmarkFragment_IntersectionCount(b *testing.B) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() f.MaxOpN = math.MaxInt32 @@ -1078,55 +1070,55 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { } func TestFragment_Tanimoto(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() - src := pilosa.NewRow(1, 2, 3) + src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 2, 200) - f.MustSetBits(101, 1, 3) - f.MustSetBits(102, 1, 2, 10, 12) + f.mustSetBits(100, 1, 3, 2, 200) + f.mustSetBits(101, 1, 3) + f.mustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { + if pairs, err := f.Top(TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 100, Count: 3}) { + } else if pairs[0] != (Pair{ID: 100, Count: 3}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 101, Count: 2}) { + } else if pairs[1] != (Pair{ID: 101, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) } } func TestFragment_Zero_Tanimoto(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() - src := pilosa.NewRow(1, 2, 3) + src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 2, 200) - f.MustSetBits(101, 1, 3) - f.MustSetBits(102, 1, 2, 10, 12) + f.mustSetBits(100, 1, 3, 2, 200) + f.mustSetBits(101, 1, 3) + f.mustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { + if pairs, err := f.Top(TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 3 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 100, Count: 3}) { + } else if pairs[0] != (Pair{ID: 100, Count: 3}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 101, Count: 2}) { + } else if pairs[1] != (Pair{ID: 101, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) - } else if pairs[2] != (pilosa.Pair{ID: 102, Count: 2}) { + } else if pairs[2] != (Pair{ID: 102, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[2]) } } func TestFragment_Snapshot_Run(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set bits on the fragment. @@ -1144,7 +1136,7 @@ func TestFragment_Snapshot_Run(t *testing.T) { } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) } else if n := f.Row(1000).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) @@ -1158,7 +1150,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { b.ReportAllocs() // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) + f := NewFragment(*FragmentPath, "i", "f", ViewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -1177,7 +1169,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { } func BenchmarkFragment_FullSnapshot(b *testing.B) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Generate some intersecting data. maxX := 1048576 / 2 @@ -1214,7 +1206,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { } func BenchmarkFragment_Import(b *testing.B) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() maxX := 1048576 * 5 * 2 sz := maxX @@ -1241,3 +1233,48 @@ func BenchmarkFragment_Import(b *testing.B) { } } } + +///////////////////////////////////////////////////////////////////// + +// mustOpenFragment returns a new instance of Fragment with a temporary path. +func mustOpenFragment(index, field, view string, slice uint64, cacheType string) *Fragment { + file, err := ioutil.TempFile("", "pilosa-fragment-") + if err != nil { + panic(err) + } + file.Close() + + if cacheType == "" { + cacheType = DefaultCacheType + } + + f := NewFragment(file.Name(), index, field, view, slice) + f.CacheType = cacheType + f.RowAttrStore = newMemAttrStore() + + if err := f.Open(); err != nil { + panic(err) + } + return f +} + +// Reopen closes the fragment and reopens it as a new instance. +func (f *Fragment) reopen() error { + if err := f.Close(); err != nil { + return err + } + if err := f.Open(); err != nil { + return err + } + return nil +} + +// mustSetBits sets columns on a row. Panic on error. +// This function does not accept a timestamp or quantum. +func (f *Fragment) mustSetBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := f.SetBit(rowID, columnID); err != nil { + panic(err) + } + } +} diff --git a/index_internal_test.go b/index_internal_test.go new file mode 100644 index 000000000..1e6d592ab --- /dev/null +++ b/index_internal_test.go @@ -0,0 +1,46 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io/ioutil" +) + +// mustOpenIndex returns a new, opened index at a temporary path. Panic on error. +func mustOpenIndex() *Index { + path, err := ioutil.TempDir("", "pilosa-index-") + if err != nil { + panic(err) + } + index, err := NewIndex(path, "i") + if err != nil { + panic(err) + } + if err := index.Open(); err != nil { + panic(err) + } + return index +} + +// reopen closes the index and reopens it. +func (i *Index) reopen() error { + if err := i.Close(); err != nil { + return err + } + if err := i.Open(); err != nil { + return err + } + return nil +} diff --git a/index_test.go b/index_test.go index 6f067f0e7..5beb67f00 100644 --- a/index_test.go +++ b/index_test.go @@ -23,6 +23,9 @@ import ( "github.com/pilosa/pilosa/test" ) +// SliceWidth is a helper reference to use when testing. +const SliceWidth = pilosa.SliceWidth + // Ensure index can open and retrieve a field. func TestIndex_CreateFieldIfNotExists(t *testing.T) { index := test.MustOpenIndex() diff --git a/test/fragment.go b/test/fragment.go index 1280d3801..140d394a7 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -15,9 +15,6 @@ package test import ( - "io/ioutil" - "os" - "github.com/pilosa/pilosa" ) @@ -30,61 +27,6 @@ type Fragment struct { RowAttrStore pilosa.AttrStore } -// NewFragment returns a new instance of Fragment with a temporary path. -func NewFragment(index, field, view string, slice uint64, cacheType string) *Fragment { - file, err := ioutil.TempFile("", "pilosa-fragment-") - if err != nil { - panic(err) - } - file.Close() - - f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), index, field, view, slice), - RowAttrStore: MustOpenAttrStore(), - } - f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore - return f -} - -// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error. -func MustOpenFragment(index, field, view string, slice uint64, cacheType string) *Fragment { - if cacheType == "" { - cacheType = pilosa.DefaultCacheType - } - f := NewFragment(index, field, view, slice, cacheType) - - if err := f.Open(); err != nil { - panic(err) - } - return f -} - -// Close closes the fragment and removes all underlying data. -func (f *Fragment) Close() error { - defer os.Remove(f.Path()) - defer os.Remove(f.CachePath()) - defer f.RowAttrStore.Close() - return f.Fragment.Close() -} - -// Reopen closes the fragment and reopens it as a new instance. -func (f *Fragment) Reopen() error { - cacheType := f.Fragment.CacheType - path := f.Path() - if err := f.Fragment.Close(); err != nil { - return err - } - - f.Fragment = pilosa.NewFragment(path, f.Index(), f.Field(), f.View(), f.Slice()) - f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore - if err := f.Open(); err != nil { - return err - } - return nil -} - // MustSetBits sets columns on a row. Panic on error. // This function does not accept a timestamp or quantum. func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { @@ -94,48 +36,3 @@ func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { } } } - -// MustClearColumns clears columns on a row. Panic on error. -func (f *Fragment) MustClearColumns(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := f.ClearBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// RowAttrStore provides simple storage for attributes. -type RowAttrStore struct { - attrs map[uint64]map[string]interface{} -} - -// NewRowAttrStore returns a new instance of RowAttrStore. -func NewRowAttrStore() *RowAttrStore { - return &RowAttrStore{ - attrs: make(map[uint64]map[string]interface{}), - } -} - -// RowAttrs returns the attributes set to a row id. -func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) { - return s.attrs[id], nil -} - -// SetRowAttrs assigns a set of attributes to a row id. -func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) { - s.attrs[id] = m -} - -// GenerateImportFill generates a set of row/col pairs that evenly fill a fragment chunk. -func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) { - ipct := int(pct * 100) - for i := 0; i < SliceWidth*rowN; i++ { - if i%100 >= ipct { - continue - } - - rowIDs = append(rowIDs, uint64(i%SliceWidth)) - columnIDs = append(columnIDs, uint64(i/SliceWidth)) - } - return -} From 15cb391570cc4b59ba6fa77d4165e523d1427061 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 7 Jun 2018 11:49:39 -0500 Subject: [PATCH 53/78] first pass at un-exporting Fragment methods --- api.go | 4 +- client.go | 2 +- executor.go | 20 ++--- field.go | 4 +- fragment.go | 170 ++++++++++++++++-------------------- fragment_internal_test.go | 176 +++++++++++++++++++------------------- holder.go | 6 +- view.go | 22 ++--- 8 files changed, 189 insertions(+), 215 deletions(-) diff --git a/api.go b/api.go index 56eaece56..3edb1b915 100644 --- a/api.go +++ b/api.go @@ -306,7 +306,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin cw := csv.NewWriter(w) // Iterate over each column. - if err := f.ForEachBit(func(rowID, columnID uint64) error { + if err := f.forEachBit(func(rowID, columnID uint64) error { return cw.Write([]string{ strconv.FormatUint(rowID, 10), strconv.FormatUint(columnID, 10), @@ -403,7 +403,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } var resp = internal.BlockDataResponse{} - resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) + resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block)) // Encode response. buf, err := proto.Marshal(&resp) diff --git a/client.go b/client.go index 81f750a39..d6c564204 100644 --- a/client.go +++ b/client.go @@ -1017,7 +1017,7 @@ type BitsByPos []Bit func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p BitsByPos) Len() int { return len(p) } func (p BitsByPos) Less(i, j int) bool { - p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID) + p0, p1 := pos(p[i].RowID, p[i].ColumnID), pos(p[j].RowID, p[j].ColumnID) if p0 == p1 { return p[i].Timestamp < p[j].Timestamp } diff --git a/executor.go b/executor.go index 411039d40..3d4dc55c1 100644 --- a/executor.go +++ b/executor.go @@ -384,7 +384,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq return ValCount{}, nil } - vsum, vcount, err := fragment.Sum(filter, bsig.BitDepth()) + vsum, vcount, err := fragment.sum(filter, bsig.BitDepth()) if err != nil { return ValCount{}, errors.Wrap(err, "computing sum") } @@ -422,7 +422,7 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fmin, fcount, err := fragment.Min(filter, bsig.BitDepth()) + fmin, fcount, err := fragment.min(filter, bsig.BitDepth()) if err != nil { return ValCount{}, err } @@ -460,7 +460,7 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fmax, fcount, err := fragment.Max(filter, bsig.BitDepth()) + fmax, fcount, err := fragment.max(filter, bsig.BitDepth()) if err != nil { return ValCount{}, err } @@ -587,7 +587,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca if tanimotoThreshold > 100 { return nil, errors.New("Tanimoto Threshold is from 1 to 100 only") } - return f.Top(TopOptions{ + return f.top(TopOptions{ N: int(n), Src: src, RowIDs: rowIDs, @@ -793,7 +793,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, return NewRow(), nil } - return frag.NotNull(bsig.BitDepth()) + return frag.notNull(bsig.BitDepth()) } else if cond.Op == pql.BETWEEN { @@ -831,10 +831,10 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, // If the query is asking for the entire valid range, just return // the not-null bitmap for the bsiGroup. if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max { - return frag.NotNull(bsig.BitDepth()) + return frag.notNull(bsig.BitDepth()) } - return frag.RangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) + return frag.rangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) } else { @@ -864,16 +864,16 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { - return frag.NotNull(bsig.BitDepth()) + return frag.notNull(bsig.BitDepth()) } // outOfRange for NEQ should return all not-null. if outOfRange && cond.Op == pql.NEQ { - return frag.NotNull(bsig.BitDepth()) + return frag.notNull(bsig.BitDepth()) } f.Stats.Count("range:bsigroup", 1, 1.0) - return frag.RangeOp(cond.Op, bsig.BitDepth(), baseValue) + return frag.rangeOp(cond.Op, bsig.BitDepth(), baseValue) } } diff --git a/field.go b/field.go index 0e1f79d66..58a65939e 100644 --- a/field.go +++ b/field.go @@ -905,7 +905,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro return errors.Wrap(err, "creating view") } - if err := frag.Import(data.RowIDs, data.ColumnIDs); err != nil { + if err := frag.bulkImport(data.RowIDs, data.ColumnIDs); err != nil { return err } } @@ -962,7 +962,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { baseValues[i] = uint64(value - bsig.Min) } - if err := frag.ImportValue(data.ColumnIDs, baseValues, bsig.BitDepth()); err != nil { + if err := frag.importValue(data.ColumnIDs, baseValues, bsig.BitDepth()); err != nil { return err } } diff --git a/fragment.go b/fragment.go index 1dfa9d09f..63d982dbe 100644 --- a/fragment.go +++ b/fragment.go @@ -130,27 +130,8 @@ func NewFragment(path, index, field, view string, slice uint64) *Fragment { } } -// Path returns the path the fragment was initialized with. -func (f *Fragment) Path() string { return f.path } - -// CachePath returns the path to the fragment's cache data. -func (f *Fragment) CachePath() string { return f.path + CacheExt } - -// Index returns the index that the fragment was initialized with. -func (f *Fragment) Index() string { return f.index } - -// Field returns the field the fragment was initialized with. -func (f *Fragment) Field() string { return f.field } - -// View returns the view the fragment was initialized with. -func (f *Fragment) View() string { return f.view } - -// Slice returns the slice the fragment was initialized with. -func (f *Fragment) Slice() uint64 { return f.slice } - -// Cache returns the fragment's cache. -// This is not safe for concurrent use. -func (f *Fragment) Cache() Cache { return f.cache } +// cachePath returns the path to the fragment's cache data. +func (f *Fragment) cachePath() string { return f.path + CacheExt } // Open opens the underlying storage. func (f *Fragment) Open() error { @@ -261,7 +242,7 @@ func (f *Fragment) openCache() error { } // Read cache data from disk. - path := f.CachePath() + path := f.cachePath() buf, err := ioutil.ReadFile(path) if os.IsNotExist(err) { return nil @@ -486,8 +467,8 @@ func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { return f.storage.Contains(pos), nil } -// Value uses a column of bits to read a multi-bit value. -func (f *Fragment) Value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +// value uses a column of bits to read a multi-bit value. +func (f *Fragment) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { f.mu.Lock() defer f.mu.Unlock() @@ -510,8 +491,8 @@ func (f *Fragment) Value(columnID uint64, bitDepth uint) (value uint64, exists b return value, true, nil } -// SetValue uses a column of bits to set a multi-bit value. -func (f *Fragment) SetValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +// setValue uses a column of bits to set a multi-bit value. +func (f *Fragment) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() @@ -582,9 +563,9 @@ func (f *Fragment) importSetValue(columnID uint64, bitDepth uint, value uint64) return changed, nil } -// Sum returns the sum of a given bsiGroup as well as the number of columns involved. +// sum returns the sum of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *Fragment) Sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { +func (f *Fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { // Compute count based on the existence row. row := f.Row(uint64(bitDepth)) if filter != nil { @@ -614,9 +595,9 @@ func (f *Fragment) Sum(filter *Row, bitDepth uint) (sum, count uint64, err error return sum, count, nil } -// Min returns the min of a given bsiGroup as well as the number of columns involved. +// min returns the min of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *Fragment) Min(filter *Row, bitDepth uint) (min, count uint64, err error) { +func (f *Fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error) { consider := f.Row(uint64(bitDepth)) if filter != nil { @@ -647,9 +628,9 @@ func (f *Fragment) Min(filter *Row, bitDepth uint) (min, count uint64, err error return min, count, nil } -// Max returns the max of a given bsiGroup as well as the number of columns involved. +// max returns the max of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *Fragment) Max(filter *Row, bitDepth uint) (max, count uint64, err error) { +func (f *Fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error) { consider := f.Row(uint64(bitDepth)) if filter != nil { @@ -678,8 +659,8 @@ func (f *Fragment) Max(filter *Row, bitDepth uint) (max, count uint64, err error return max, count, nil } -// RangeOp returns bitmaps with a bsiGroup value encoding matching the predicate. -func (f *Fragment) RangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +// rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate. +func (f *Fragment) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { switch op { case pql.EQ: return f.rangeEQ(bitDepth, predicate) @@ -812,13 +793,13 @@ func (f *Fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) return b, nil } -// NotNull returns the not-null row (stored at bitDepth). -func (f *Fragment) NotNull(bitDepth uint) (*Row, error) { +// notNull returns the not-null row (stored at bitDepth). +func (f *Fragment) notNull(bitDepth uint) (*Row, error) { return f.Row(uint64(bitDepth)), nil } -// RangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. -func (f *Fragment) RangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { +// rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. +func (f *Fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { b := f.Row(uint64(bitDepth)) keep1 := NewRow() // GTE keep2 := NewRow() // LTE @@ -864,12 +845,12 @@ func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { if columnID < minColumnID || columnID >= minColumnID+SliceWidth { return 0, errors.New("column out of bounds") } - return Pos(rowID, columnID), nil + return pos(rowID, columnID), nil } -// ForEachBit executes fn for every bit set in the fragment. +// forEachBit executes fn for every bit set in the fragment. // Errors returned from fn are passed through. -func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { +func (f *Fragment) forEachBit(fn func(rowID, columnID uint64) error) error { f.mu.Lock() defer f.mu.Unlock() @@ -886,10 +867,10 @@ func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { return err } -// Top returns the top rows from the fragment. +// top returns the top rows from the fragment. // If opt.Src is specified then only rows which intersect src are returned. // If opt.FilterValues exist then the row attribute specified by field is matched. -func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { +func (f *Fragment) top(opt TopOptions) ([]Pair, error) { // Retrieve pairs. If no row ids specified then return from cache. pairs := f.topBitmapPairs(opt.RowIDs) @@ -1089,13 +1070,6 @@ func (f *Fragment) Checksum() []byte { return h.Sum(nil) } -// BlockN returns the number of blocks in the fragment. -func (f *Fragment) BlockN() int { - f.mu.Lock() - defer f.mu.Unlock() - return int(f.storage.Max() / (HashBlockSize * SliceWidth)) -} - // InvalidateChecksums clears all cached block checksums. func (f *Fragment) InvalidateChecksums() { f.mu.Lock() @@ -1184,8 +1158,8 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i } } -// BlockData returns bits in a block as row & column ID pairs. -func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { +// blockData returns bits in a block as row & column ID pairs. +func (f *Fragment) blockData(id int) (rowIDs, columnIDs []uint64) { f.mu.Lock() defer f.mu.Unlock() @@ -1196,17 +1170,17 @@ func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { return } -// MergeBlock compares the block's bits and computes a diff with another set of block bits. +// mergeBlock compares the block's bits and computes a diff with another set of block bits. // The state of a bit is determined by consensus from all blocks being considered. // // For example, if 3 blocks are compared and two have a set bit and one has a // cleared bit then the bit is considered cleared. The function returns the // diff per incoming block so that all can be in sync. -func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) { +func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, err error) { // Ensure that all pair sets are of equal length. for i := range data { - if len(data[i].RowIDs) != len(data[i].ColumnIDs) { - return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].RowIDs), len(data[i].ColumnIDs)) + if len(data[i].rowIDs) != len(data[i].columnIDs) { + return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].rowIDs), len(data[i].columnIDs)) } } @@ -1214,8 +1188,8 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e defer f.mu.Unlock() // Track sets and clears for all blocks (including local). - sets = make([]PairSet, len(data)+1) - clears = make([]PairSet, len(data)+1) + sets = make([]pairSet, len(data)+1) + clears = make([]pairSet, len(data)+1) // Limit upper row/column pair. maxRowID := uint64(id+1) * HashBlockSize @@ -1231,7 +1205,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e // Append buffered iterators for each incoming block. for i := range data { - var itr Iterator = NewSliceIterator(data[i].RowIDs, data[i].ColumnIDs) + var itr Iterator = NewSliceIterator(data[i].rowIDs, data[i].columnIDs) itr = NewLimitIterator(itr, maxRowID, maxColumnID) itrs = append(itrs, NewBufIterator(itr)) } @@ -1296,25 +1270,25 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e // Append to either the set or clear diff. if newValue { - sets[i].RowIDs = append(sets[i].RowIDs, min.rowID) - sets[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) + sets[i].rowIDs = append(sets[i].rowIDs, min.rowID) + sets[i].columnIDs = append(sets[i].columnIDs, min.columnID) } else { - clears[i].RowIDs = append(sets[i].RowIDs, min.rowID) - clears[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) + clears[i].rowIDs = append(sets[i].rowIDs, min.rowID) + clears[i].columnIDs = append(sets[i].columnIDs, min.columnID) } } } // Set local bits. - for i := range sets[0].ColumnIDs { - if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil { + for i := range sets[0].columnIDs { + if _, err := f.setBit(sets[0].rowIDs[i], (f.slice*SliceWidth)+sets[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "setting") } } // Clear local bits. - for i := range clears[0].ColumnIDs { - if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil { + for i := range clears[0].columnIDs { + if _, err := f.clearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "clearing") } } @@ -1322,9 +1296,9 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e return sets[1:], clears[1:], nil } -// Import bulk imports a set of bits and then snapshots the storage. +// bulkImport bulk imports a set of bits and then snapshots the storage. // This does not affect the fragment's cache. -func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { +func (f *Fragment) bulkImport(rowIDs, columnIDs []uint64) error { f.mu.Lock() defer f.mu.Unlock() // Verify that there are an equal number of row ids and column ids. @@ -1392,8 +1366,8 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { return nil } -// ImportValue bulk imports a set of range-encoded values. -func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error { +// importValue bulk imports a set of range-encoded values. +func (f *Fragment) importValue(columnIDs, values []uint64, bitDepth uint) error { f.mu.Lock() defer f.mu.Unlock() // Verify that there are an equal number of column ids and values. @@ -1529,7 +1503,7 @@ func (f *Fragment) flushCache() error { } // Write to disk. - if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil { + if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil { return errors.Wrap(err, "writing") } @@ -1603,7 +1577,7 @@ func (f *Fragment) writeCacheToArchive(tw *tar.Writer) error { defer f.mu.Unlock() // Read cache into buffer. - buf, err := ioutil.ReadFile(f.CachePath()) + buf, err := ioutil.ReadFile(f.cachePath()) if os.IsNotExist(err) { return nil } else if err != nil { @@ -1697,7 +1671,7 @@ func (f *Fragment) readCacheFromArchive(r io.Reader) error { buf, err := ioutil.ReadAll(r) if err != nil { return errors.Wrap(err, "reading") - } else if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil { + } else if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil { return errors.Wrap(err, "writing") } @@ -1762,11 +1736,11 @@ func (s *FragmentSyncer) isClosing() bool { } } -// SyncFragment compares checksums for the local and remote fragments and +// syncFragment compares checksums for the local and remote fragments and // then merges any blocks which have differences. -func (s *FragmentSyncer) SyncFragment() error { +func (s *FragmentSyncer) syncFragment() error { // Determine replica set. - nodes := s.Cluster.SliceNodes(s.Fragment.Index(), s.Fragment.Slice()) + nodes := s.Cluster.SliceNodes(s.Fragment.index, s.Fragment.slice) if len(nodes) == 1 { return nil } @@ -1783,7 +1757,7 @@ func (s *FragmentSyncer) SyncFragment() error { // Retrieve remote blocks. client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Field(), s.Fragment.Slice()) + blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.index, s.Fragment.field, s.Fragment.slice) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } @@ -1846,9 +1820,9 @@ func (s *FragmentSyncer) syncBlock(id int) error { f := s.Fragment // Read pairs from each remote block. - var pairSets []PairSet + var pairSets []pairSet var clients []InternalClient - for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) { + for _, node := range s.Cluster.SliceNodes(f.index, f.slice) { if s.Node.ID == node.ID { continue } @@ -1862,14 +1836,14 @@ func (s *FragmentSyncer) syncBlock(id int) error { clients = append(clients, client) // Only sync the standard block. - rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Field(), f.Slice(), id) + rowIDs, columnIDs, err := client.BlockData(context.Background(), f.index, f.field, f.slice, id) if err != nil { return errors.Wrap(err, "getting block") } - pairSets = append(pairSets, PairSet{ - ColumnIDs: columnIDs, - RowIDs: rowIDs, + pairSets = append(pairSets, pairSet{ + columnIDs: columnIDs, + rowIDs: rowIDs, }) } @@ -1879,7 +1853,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { } // Merge blocks together. - sets, clears, err := f.MergeBlock(id, pairSets) + sets, clears, err := f.mergeBlock(id, pairSets) if err != nil { return errors.Wrap(err, "merging") } @@ -1890,12 +1864,12 @@ func (s *FragmentSyncer) syncBlock(id int) error { count := 0 // Ignore if there are no differences. - if len(set.ColumnIDs) == 0 && len(clear.ColumnIDs) == 0 { + if len(set.columnIDs) == 0 && len(clear.columnIDs) == 0 { continue } // Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest. - total := len(set.ColumnIDs) + len(clear.ColumnIDs) + total := len(set.columnIDs) + len(clear.columnIDs) maxWrites := s.Cluster.MaxWritesPerRequest if maxWrites <= 0 { maxWrites = 5000 @@ -1903,12 +1877,12 @@ func (s *FragmentSyncer) syncBlock(id int) error { buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites)))) // Only sync the standard block. - for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.Field(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + for j := 0; j < len(set.columnIDs); j++ { + fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.field, set.rowIDs[j], (f.slice*SliceWidth)+set.columnIDs[j]) count++ } - for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.Field(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + for j := 0; j < len(clear.columnIDs); j++ { + fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.field, clear.rowIDs[j], (f.slice*SliceWidth)+clear.columnIDs[j]) count++ } @@ -1924,7 +1898,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { Query: buffers[k].String(), Remote: true, } - _, err := clients[i].Query(context.Background(), f.Index(), queryRequest) + _, err := clients[i].Query(context.Background(), f.index, queryRequest) if err != nil { return errors.Wrap(err, "executing") } @@ -1942,10 +1916,10 @@ func madvise(b []byte, advice int) (err error) { return } -// PairSet is a list of equal length row and column id lists. -type PairSet struct { - RowIDs []uint64 - ColumnIDs []uint64 +// pairSet is a list of equal length row and column id lists. +type pairSet struct { + rowIDs []uint64 + columnIDs []uint64 } // byteSlicesEqual returns true if all slices are equal. @@ -1962,7 +1936,7 @@ func byteSlicesEqual(a [][]byte) bool { return true } -// Pos returns the row position of a row/column pair. -func Pos(rowID, columnID uint64) uint64 { +// pos returns the row position of a row/column pair. +func pos(rowID, columnID uint64) uint64 { return (rowID * SliceWidth) + (columnID % SliceWidth) } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 20f3f7b6a..496a2888a 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -99,14 +99,14 @@ func TestFragment_SetValue(t *testing.T) { defer f.Close() // Set value. - if changed, err := f.SetValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.Value(100, 16); err != nil { + if value, exists, err := f.value(100, 16); err != nil { t.Fatal(err) } else if value != 3829 { t.Fatalf("unexpected value: %d", value) @@ -115,7 +115,7 @@ func TestFragment_SetValue(t *testing.T) { } // Setting value should return no change. - if changed, err := f.SetValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if changed { t.Fatal("expected no change") @@ -127,21 +127,21 @@ func TestFragment_SetValue(t *testing.T) { defer f.Close() // Set value. - if changed, err := f.SetValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Overwriting value should overwrite all bits. - if changed, err := f.SetValue(100, 16, 2028); err != nil { + if changed, err := f.setValue(100, 16, 2028); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.Value(100, 16); err != nil { + if value, exists, err := f.value(100, 16); err != nil { t.Fatal(err) } else if value != 2028 { t.Fatalf("unexpected value: %d", value) @@ -155,14 +155,14 @@ func TestFragment_SetValue(t *testing.T) { defer f.Close() // Set value. - if changed, err := f.SetValue(100, 10, 20); err != nil { + if changed, err := f.setValue(100, 10, 20); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Non-existent value. - if value, exists, err := f.Value(100, 11); err != nil { + if value, exists, err := f.value(100, 11); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -191,14 +191,14 @@ func TestFragment_SetValue(t *testing.T) { m[columnID] = int64(value) - if _, err := f.SetValue(columnID, bitDepth, value); err != nil { + if _, err := f.setValue(columnID, bitDepth, value); err != nil { t.Fatal(err) } } // Ensure values are set. for columnID, value := range m { - v, exists, err := f.Value(columnID, bitDepth) + v, exists, err := f.value(columnID, bitDepth) if err != nil { t.Fatal(err) } else if value != int64(v) { @@ -223,18 +223,18 @@ func TestFragment_Sum(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } t.Run("NoFilter", func(t *testing.T) { - if sum, n, err := f.Sum(nil, bitDepth); err != nil { + if sum, n, err := f.sum(nil, bitDepth); err != nil { t.Fatal(err) } else if n != 4 { t.Fatalf("unexpected count: %d", n) @@ -244,7 +244,7 @@ func TestFragment_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if sum, n, err := f.Sum(NewRow(2000, 4000, 5000), bitDepth); err != nil { + if sum, n, err := f.sum(NewRow(2000, 4000, 5000), bitDepth); err != nil { t.Fatal(err) } else if n != 2 { t.Fatalf("unexpected count: %d", n) @@ -262,19 +262,19 @@ func TestFragment_MinMax(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(5000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(6000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(7000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(7000, bitDepth, 0); err != nil { t.Fatal(err) } @@ -292,7 +292,7 @@ func TestFragment_MinMax(t *testing.T) { {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if min, cnt, err := f.Min(test.filter, bitDepth); err != nil { + if min, cnt, err := f.min(test.filter, bitDepth); err != nil { t.Fatal(err) } else if min != test.exp { t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) @@ -316,7 +316,7 @@ func TestFragment_MinMax(t *testing.T) { {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if max, cnt, err := f.Max(test.filter, bitDepth); err != nil { + if max, cnt, err := f.max(test.filter, bitDepth); err != nil { t.Fatal(err) } else if max != test.exp { t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) @@ -336,18 +336,18 @@ func TestFragment_Range(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for equality. - if b, err := f.RangeOp(pql.EQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.EQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -359,18 +359,18 @@ func TestFragment_Range(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for inequality. - if b, err := f.RangeOp(pql.NEQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.NEQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -382,43 +382,43 @@ func TestFragment_Range(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values less than (ending with set column). - if b, err := f.RangeOp(pql.LT, bitDepth, 301); err != nil { + if b, err := f.rangeOp(pql.LT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than (ending with unset column). - if b, err := f.RangeOp(pql.LT, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.LT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than or equal to (ending with set column). - if b, err := f.RangeOp(pql.LTE, bitDepth, 301); err != nil { + if b, err := f.rangeOp(pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than or equal to (ending with unset column). - if b, err := f.RangeOp(pql.LTE, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -430,43 +430,43 @@ func TestFragment_Range(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values greater than (ending with unset bit). - if b, err := f.RangeOp(pql.GT, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than (ending with set bit). - if b, err := f.RangeOp(pql.GT, bitDepth, 301); err != nil { + if b, err := f.rangeOp(pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with unset bit). - if b, err := f.RangeOp(pql.GTE, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with set bit). - if b, err := f.RangeOp(pql.GTE, bitDepth, 301); err != nil { + if b, err := f.rangeOp(pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -478,43 +478,43 @@ func TestFragment_Range(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values greater than (ending with unset column). - if b, err := f.RangeBetween(bitDepth, 300, 2817); err != nil { + if b, err := f.rangeBetween(bitDepth, 300, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than (ending with set column). - if b, err := f.RangeBetween(bitDepth, 301, 2817); err != nil { + if b, err := f.rangeBetween(bitDepth, 301, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with unset column). - if b, err := f.RangeBetween(bitDepth, 301, 2816); err != nil { + if b, err := f.rangeBetween(bitDepth, 301, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with set column). - if b, err := f.RangeBetween(bitDepth, 300, 2816); err != nil { + if b, err := f.rangeBetween(bitDepth, 300, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -567,7 +567,7 @@ func TestFragment_ForEachBit(t *testing.T) { // Iterate over bits. var result [][2]uint64 - if err := f.ForEachBit(func(rowID, columnID uint64) error { + if err := f.forEachBit(func(rowID, columnID uint64) error { result = append(result, [2]uint64{rowID, columnID}) return nil }); err != nil { @@ -591,7 +591,7 @@ func TestFragment_Top(t *testing.T) { f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(TopOptions{N: 2}); err != nil { + if pairs, err := f.top(TopOptions{N: 2}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -617,7 +617,7 @@ func TestFragment_Top_Filter(t *testing.T) { f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": int64(20)}) // Retrieve top rows. - if pairs, err := f.Top(TopOptions{ + if pairs, err := f.top(TopOptions{ N: 2, FilterName: "x", FilterValues: []interface{}{int64(10), int64(15), int64(20)}, @@ -648,7 +648,7 @@ func TestFragment_TopN_Intersect(t *testing.T) { f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(TopOptions{N: 3, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{N: 3, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 3}, @@ -683,7 +683,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(TopOptions{N: 10, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{N: 10, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 999, Count: 19}, @@ -712,7 +712,7 @@ func TestFragment_TopN_IDs(t *testing.T) { f.mustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.Top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 4}, @@ -733,7 +733,7 @@ func TestFragment_TopN_NopCache(t *testing.T) { f.mustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.Top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{}) { t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) @@ -792,7 +792,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } // Retrieve top rows. - if pairs, err := f.Top(TopOptions{N: 5}); err != nil { + if pairs, err := f.top(TopOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) > int(cacheSize) { t.Fatalf("TopN count cannot exceed cache size: %d", cacheSize) @@ -891,8 +891,8 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { } // Verify correct cache type and size. - if cache, ok := f.Cache().(*LRUCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*LRUCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -903,8 +903,8 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { } // Re-verify correct cache type and size. - if cache, ok := f.Cache().(*LRUCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*LRUCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -941,8 +941,8 @@ func TestFragment_RankCache_Persistence(t *testing.T) { } // Verify correct cache type and size. - if cache, ok := f.Cache().(*RankCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*RankCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -956,8 +956,8 @@ func TestFragment_RankCache_Persistence(t *testing.T) { f = index.Field("f").View(ViewStandard).Fragment(0) // Re-verify correct cache type and size. - if cache, ok := f.Cache().(*RankCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*RankCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -978,7 +978,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Verify cache is populated. - if n := f0.Cache().Len(); n != 1 { + if n := f0.cache.Len(); n != 1 { t.Fatalf("unexpected cache size: %d", n) } @@ -998,7 +998,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Verify cache is in other fragment. - if n := f1.Cache().Len(); n != 1 { + if n := f1.cache.Len(); n != 1 { t.Fatalf("unexpected cache size: %d", n) } @@ -1010,7 +1010,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { // Close and reopen the fragment & verify the data. if err := f1.reopen(); err != nil { t.Fatal(err) - } else if n := f1.Cache().Len(); n != 1 { + } else if n := f1.cache.Len(); n != 1 { t.Fatalf("unexpected cache size (reopen): %d", n) } else if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected columns (reopen): %+v", a) @@ -1081,7 +1081,7 @@ func TestFragment_Tanimoto(t *testing.T) { f.mustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.Top(TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -1104,7 +1104,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { f.mustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.Top(TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 3 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -1187,7 +1187,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { val += 2 i++ } - if err := f.Import(rows, cols); err != nil { + if err := f.bulkImport(rows, cols); err != nil { b.Fatalf("Error Building Sample: %s", err) } if row > max { @@ -1228,7 +1228,7 @@ func BenchmarkFragment_Import(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - if err := f.Import(rows, cols); err != nil { + if err := f.bulkImport(rows, cols); err != nil { b.Fatalf("Error Building Sample: %s", err) } } diff --git a/holder.go b/holder.go index e8db140ce..89bd508ac 100644 --- a/holder.go +++ b/holder.go @@ -445,7 +445,7 @@ func (h *Holder) flushCaches() { } if err := fragment.FlushCache(); err != nil { - h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.cachePath()) } } } @@ -765,7 +765,7 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err Closing: s.Closing, RemoteClient: s.RemoteClient, } - if err := fs.SyncFragment(); err != nil { + if err := fs.syncFragment(); err != nil { return errors.Wrap(err, "syncing fragment") } @@ -809,7 +809,7 @@ func (c *HolderCleaner) CleanHolder() error { for _, field := range index.Fields() { for _, view := range field.Views() { for _, fragment := range view.Fragments() { - fragSlice := fragment.Slice() + fragSlice := fragment.slice // Ignore fragments that should be present. if uint64InSlice(fragSlice, containedSlices) { continue diff --git a/view.go b/view.go index d76f33d59..06545fc84 100644 --- a/view.go +++ b/view.go @@ -151,10 +151,10 @@ func (v *View) openFragments() error { frag := v.newFragment(v.FragmentPath(slice), slice) if err := frag.Open(); err != nil { - return fmt.Errorf("open fragment: slice=%d, err=%s", frag.Slice(), err) + return fmt.Errorf("open fragment: slice=%d, err=%s", frag.slice, err) } frag.RowAttrStore = v.RowAttrStore - v.fragments[frag.Slice()] = frag + v.fragments[frag.slice] = frag } return nil @@ -289,12 +289,12 @@ func (v *View) DeleteFragment(slice uint64) error { } // Delete fragment file. - if err := os.Remove(fragment.Path()); err != nil { + if err := os.Remove(fragment.path); err != nil { return errors.Wrap(err, "deleting fragment file") } // Delete fragment cache file. - if err := os.Remove(fragment.CachePath()); err != nil { + if err := os.Remove(fragment.cachePath()); err != nil { v.Logger.Printf("no cache file to delete for slice %d", slice) } @@ -330,7 +330,7 @@ func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool, if err != nil { return value, exists, err } - return frag.Value(columnID, bitDepth) + return frag.value(columnID, bitDepth) } // setValue uses a column of bits to set a multi-bit value. @@ -340,13 +340,13 @@ func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed b if err != nil { return changed, err } - return frag.SetValue(columnID, bitDepth, value) + return frag.setValue(columnID, bitDepth, value) } // sum returns the sum & count of a field. func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { for _, f := range v.Fragments() { - fsum, fcount, err := f.Sum(filter, bitDepth) + fsum, fcount, err := f.sum(filter, bitDepth) if err != nil { return sum, count, err } @@ -360,7 +360,7 @@ func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) { var minHasValue bool for _, f := range v.Fragments() { - fmin, fcount, err := f.Min(filter, bitDepth) + fmin, fcount, err := f.min(filter, bitDepth) if err != nil { return min, count, err } @@ -387,7 +387,7 @@ func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) { // max returns the max and count of a field. func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) { for _, f := range v.Fragments() { - fmax, fcount, err := f.Max(filter, bitDepth) + fmax, fcount, err := f.max(filter, bitDepth) if err != nil { return max, count, err } @@ -403,7 +403,7 @@ func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) { func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { r := NewRow() for _, frag := range v.Fragments() { - other, err := frag.RangeOp(op, bitDepth, predicate) + other, err := frag.rangeOp(op, bitDepth, predicate) if err != nil { return nil, err } @@ -417,7 +417,7 @@ func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, err func (v *View) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { r := NewRow() for _, frag := range v.Fragments() { - other, err := frag.RangeBetween(bitDepth, predicateMin, predicateMax) + other, err := frag.rangeBetween(bitDepth, predicateMin, predicateMax) if err != nil { return nil, err } From 468ad57b6da2b9c65052ec61974bda48a4624818 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 7 Jun 2018 12:11:12 -0500 Subject: [PATCH 54/78] un-export Fragment.SetBit and Fragment.ClearBit. adds methods to test.Holder to set/clear bits on a field. --- client_test.go | 42 ++++++------ executor_test.go | 134 +++++++++++++++++++------------------- fragment.go | 26 ++++---- fragment_internal_test.go | 54 +++++++-------- handler_test.go | 12 ++-- holder_test.go | 91 +++++++------------------- stats_test.go | 30 ++++----- test/fragment.go | 10 --- test/holder.go | 28 ++++++++ view.go | 4 +- view_test.go | 10 --- 11 files changed, 202 insertions(+), 239 deletions(-) diff --git a/client_test.go b/client_test.go index d2e194805..2427fe6c2 100644 --- a/client_test.go +++ b/client_test.go @@ -109,26 +109,26 @@ func TestClient_MultiNode(t *testing.T) { } } - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10) + hldr[0].MustSetBits("i", "f", 100, baseBit0+10) + hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12) + hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) + hldr[0].MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) + hldr[0].MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) + hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2, baseBit0+10) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) + hldr[1].MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) + hldr[1].MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) + hldr[1].MustSetBits("i", "f", 98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) + hldr[1].MustSetBits("i", "f", 1, baseBit1+4) + hldr[1].MustSetBits("i", "f", 22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12) + hldr[2].MustSetBits("i", "f", 24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) + hldr[2].MustSetBits("i", "f", 20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) + hldr[2].MustSetBits("i", "f", 21, baseBit2+10) + hldr[2].MustSetBits("i", "f", 100, baseBit2+10) + hldr[2].MustSetBits("i", "f", 99, baseBit2+10, baseBit2+11, baseBit2+12) + hldr[2].MustSetBits("i", "f", 98, baseBit2+10, baseBit2+11) + hldr[2].MustSetBits("i", "f", 22, baseBit2+10, baseBit2+11, baseBit2+12) // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay @@ -322,11 +322,11 @@ func TestClient_FragmentBlocks(t *testing.T) { defer hldr.Close() // Set two bits on blocks 0 & 3. - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100) // Set a bit on a different slice. - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1) + hldr.SetBit("i", "f", 0, 1) s := test.NewServer() defer s.Close() diff --git a/executor_test.go b/executor_test.go index 01c99e9c5..9cf44ca62 100644 --- a/executor_test.go +++ b/executor_test.go @@ -107,11 +107,11 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { func TestExecutor_Execute_Difference(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, 2) + hldr.SetBit("i", "general", 10, 3) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, 4) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -125,7 +125,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { func TestExecutor_Execute_Empty_Difference(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr.SetBit("i", "general", 10, 1) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil { @@ -137,13 +137,13 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { func TestExecutor_Execute_Intersect(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, SliceWidth+1) + hldr.SetBit("i", "general", 10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 11, 1) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -168,12 +168,12 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { func TestExecutor_Execute_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, SliceWidth+1) + hldr.SetBit("i", "general", 10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -187,7 +187,7 @@ func TestExecutor_Execute_Union(t *testing.T) { func TestExecutor_Execute_Empty_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr.SetBit("i", "general", 10, 0) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { @@ -201,12 +201,12 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { func TestExecutor_Execute_Xor(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, SliceWidth+1) + hldr.SetBit("i", "general", 10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -220,9 +220,9 @@ func TestExecutor_Execute_Xor(t *testing.T) { func TestExecutor_Execute_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "f", 10, 3) + hldr.SetBit("i", "f", 10, SliceWidth+1) + hldr.SetBit("i", "f", 10, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { @@ -427,12 +427,12 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { defer hldr.Close() // Set columns for rows 0, 10, & 20 across two slices. - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 0, 2) + hldr.SetBit("i", "f", 0, SliceWidth) + hldr.SetBit("i", "f", 1, SliceWidth+2) + hldr.SetBit("i", "f", 1, SliceWidth) // Execute query. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) @@ -450,23 +450,23 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, SliceWidth) + hldr.SetBit("i", "f", 0, 2*SliceWidth) + hldr.SetBit("i", "f", 0, 3*SliceWidth) + hldr.SetBit("i", "f", 0, 4*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1) + hldr.SetBit("i", "f", 1, 0) + hldr.SetBit("i", "f", 1, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1) + hldr.SetBit("i", "f", 2, SliceWidth) + hldr.SetBit("i", "f", 2, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1) + hldr.SetBit("i", "f", 3, 2*SliceWidth) + hldr.SetBit("i", "f", 3, 2*SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) + hldr.SetBit("i", "f", 4, 3*SliceWidth) + hldr.SetBit("i", "f", 4, 3*SliceWidth+1) // Execute query. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) @@ -485,19 +485,19 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { defer hldr.Close() // Set columns for rows 0, 10, & 20 across two slices. - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 0, SliceWidth) + hldr.SetBit("i", "f", 10, SliceWidth) + hldr.SetBit("i", "f", 10, SliceWidth+1) + hldr.SetBit("i", "f", 20, SliceWidth) + hldr.SetBit("i", "f", 20, SliceWidth+1) + hldr.SetBit("i", "f", 20, SliceWidth+2) // Create an intersecting row. - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) + hldr.SetBit("i", "other", 100, SliceWidth) + hldr.SetBit("i", "other", 100, SliceWidth+1) + hldr.SetBit("i", "other", 100, SliceWidth+2) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() @@ -521,9 +521,9 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { // hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 10, SliceWidth) if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) @@ -544,9 +544,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 10, SliceWidth) if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) @@ -989,7 +989,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) + hldr.SetBit("i", "f", 10, SliceWidth+1) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil { @@ -1023,8 +1023,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) + hldr.SetBit("i", "f", 10, (2*SliceWidth)+1) + hldr.SetBit("i", "f", 10, (2*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { @@ -1192,8 +1192,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) + hldr.SetBit("i", "f", 30, (2*SliceWidth)+1) + hldr.SetBit("i", "f", 30, (4*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=3)`), nil, nil); err != nil { diff --git a/fragment.go b/fragment.go index 63d982dbe..85e1c2b34 100644 --- a/fragment.go +++ b/fragment.go @@ -361,15 +361,15 @@ func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *R return row } -// SetBit sets a bit for a given column & row within the fragment. +// setBit sets a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) { +func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - return f.setBit(rowID, columnID) + return f.unprotectedSetBit(rowID, columnID) } -func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { +func (f *Fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) @@ -413,15 +413,15 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { return changed, nil } -// ClearBit clears a bit for a given column & row within the fragment. +// clearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) { +func (f *Fragment) clearBit(rowID, columnID uint64) (bool, error) { f.mu.Lock() defer f.mu.Unlock() - return f.clearBit(rowID, columnID) + return f.unprotectedClearBit(rowID, columnID) } -func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { +func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) @@ -498,13 +498,13 @@ func (f *Fragment) setValue(columnID uint64, bitDepth uint, value uint64) (chang for i := uint(0); i < bitDepth; i++ { if value&(1< Date: Thu, 7 Jun 2018 15:09:58 -0500 Subject: [PATCH 55/78] unexport Fragment.Row(). This required creating Field.Row() and View.row() --- client_test.go | 8 +++--- executor.go | 4 +-- executor_test.go | 13 +++++---- field.go | 22 +++++++++++++++ fragment.go | 52 ++++++++++++++++++------------------ fragment_internal_test.go | 26 +++++++++--------- holder_test.go | 56 ++++++++++++++++----------------------- test/holder.go | 28 ++++++++++++++++++++ view.go | 6 +++++ 9 files changed, 132 insertions(+), 83 deletions(-) diff --git a/client_test.go b/client_test.go index 2427fe6c2..6456ef386 100644 --- a/client_test.go +++ b/client_test.go @@ -211,8 +211,8 @@ func TestClient_Import(t *testing.T) { defer hldr.Close() // Load bitmap into cache to ensure cache gets updated. - f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - f.Row(0) + hldr.SetBit("i", "f", 1, 0) // set a bit so the view gets created. + hldr.Row("i", "f", 0, 0) s := test.NewServer() defer s.Close() @@ -231,10 +231,10 @@ func TestClient_Import(t *testing.T) { } // Verify data. - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { + if a := hldr.Row("i", "f", 0, 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { t.Fatalf("unexpected columns: %+v", a) } - if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { + if a := hldr.Row("i", "f", 0, 200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { t.Fatalf("unexpected columns: %+v", a) } } diff --git a/executor.go b/executor.go index 3d4dc55c1..a784a497a 100644 --- a/executor.go +++ b/executor.go @@ -649,7 +649,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. if frag == nil { return NewRow(), nil } - return frag.Row(rowID), nil + return frag.row(rowID), nil } // executeIntersectSlice executes a intersect() call for a local slice. @@ -741,7 +741,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C if f == nil { continue } - row = row.Union(f.Row(rowID)) + row = row.Union(f.row(rowID)) } f.Stats.Count("range", 1, 1.0) return row, nil diff --git a/executor_test.go b/executor_test.go index 9cf44ca62..81fc74c72 100644 --- a/executor_test.go +++ b/executor_test.go @@ -237,9 +237,11 @@ func TestExecutor_Execute_SetBit(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() + // set a bit so the view gets created. + hldr.SetBit("i", "f", 1, 0) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if n := f.Row(11).Count(); n != 0 { + if n := hldr.Row("i", "f", 0, 11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } @@ -251,7 +253,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } } - if n := f.Row(11).Count(); n != 1 { + if n := hldr.Row("i", "f", 0, 11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { @@ -1078,7 +1080,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } // Verify that one column is set on both node's holder. - if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { + if n := hldr.Row("i", "f", 0, 10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -1132,7 +1134,8 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } // Verify that one column is set on both node's holder. - if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 { + //if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 { + if n := hldr.ViewRow("i", "f", "standard_2016", 0, 10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { diff --git a/field.go b/field.go index 58a65939e..132458652 100644 --- a/field.go +++ b/field.go @@ -620,6 +620,28 @@ func (f *Field) DeleteView(name string) error { return nil } +// Row returns a row for a slice of the standard view. +func (f *Field) Row(slice, rowID uint64) (*Row, error) { + if f.Type() != FieldTypeSet { + return nil, errors.Errorf("row method unsupported for field type: %s", f.Type()) + } + view := f.View(ViewStandard) + if view == nil { + return nil, ErrInvalidView + } + return view.row(slice, rowID), nil +} + +// ViewRow returns a row for a view and slice. +// TODO: unexport this with views (it's only used in tests). +func (f *Field) ViewRow(viewName string, slice, rowID uint64) (*Row, error) { + view := f.View(viewName) + if view == nil { + return nil, ErrInvalidView + } + return view.row(slice, rowID), nil +} + // SetBit sets a bit on a view within the field. func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. diff --git a/fragment.go b/fragment.go index 85e1c2b34..268be5b70 100644 --- a/fragment.go +++ b/fragment.go @@ -323,14 +323,14 @@ func (f *Fragment) closeStorage() error { return nil } -// Row returns a row by ID. -func (f *Fragment) Row(rowID uint64) *Row { +// row returns a row by ID. +func (f *Fragment) row(rowID uint64) *Row { f.mu.Lock() defer f.mu.Unlock() - return f.row(rowID, true, true) + return f.unprotectedRow(rowID, true, true) } -func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *Row { +func (f *Fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCache bool) *Row { if checkRowCache { r, ok := f.rowCache.Fetch(rowID) if ok && r != nil { @@ -396,7 +396,7 @@ func (f *Fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err } // Get the row from row cache or fragment.storage. - row := f.row(rowID, true, true) + row := f.unprotectedRow(rowID, true, true) row.SetBit(columnID) // Update the cache. @@ -448,7 +448,7 @@ func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er } // Get the row from cache or fragment.storage. - row := f.row(rowID, true, true) + row := f.unprotectedRow(rowID, true, true) row.ClearBit(columnID) // Update the cache. @@ -567,7 +567,7 @@ func (f *Fragment) importSetValue(columnID uint64, bitDepth uint, value uint64) // A bitmap can be passed in to optionally filter the computed columns. func (f *Fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { // Compute count based on the existence row. - row := f.Row(uint64(bitDepth)) + row := f.row(uint64(bitDepth)) if filter != nil { count = row.IntersectionCount(filter) } else { @@ -582,7 +582,7 @@ func (f *Fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error // 10*(2^0) + 4*(2^1) + 3*(2^2) = 30 // for i := uint(0); i < bitDepth; i++ { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) cnt := uint64(0) if filter != nil { cnt = row.IntersectionCount(filter) @@ -599,7 +599,7 @@ func (f *Fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error // A bitmap can be passed in to optionally filter the computed columns. func (f *Fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error) { - consider := f.Row(uint64(bitDepth)) + consider := f.row(uint64(bitDepth)) if filter != nil { consider = consider.Intersect(filter) } @@ -611,7 +611,7 @@ func (f *Fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error for i := bitDepth; i > uint(0); i-- { ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.Row(uint64(ii)) + row := f.row(uint64(ii)) x := consider.Difference(row) count = x.Count() @@ -632,7 +632,7 @@ func (f *Fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error // A bitmap can be passed in to optionally filter the computed columns. func (f *Fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error) { - consider := f.Row(uint64(bitDepth)) + consider := f.row(uint64(bitDepth)) if filter != nil { consider = consider.Intersect(filter) } @@ -644,7 +644,7 @@ func (f *Fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error for i := bitDepth; i > uint(0); i-- { ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.Row(uint64(ii)) + row := f.row(uint64(ii)) x := row.Intersect(consider) count = x.Count() @@ -677,11 +677,11 @@ func (f *Fragment) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, func (f *Fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 if bit == 1 { @@ -696,7 +696,7 @@ func (f *Fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { func (f *Fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Get the equal bitmap. eq, err := f.rangeEQ(bitDepth, predicate) @@ -714,12 +714,12 @@ func (f *Fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) keep := NewRow() // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Filter any bits that don't match the current bit value. leadingZeros := true for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 // Remove any columns with higher bits set. @@ -759,12 +759,12 @@ func (f *Fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) } func (f *Fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) keep := NewRow() // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 // Handle last bit differently. @@ -795,18 +795,18 @@ func (f *Fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) // notNull returns the not-null row (stored at bitDepth). func (f *Fragment) notNull(bitDepth uint) (*Row, error) { - return f.Row(uint64(bitDepth)), nil + return f.row(uint64(bitDepth)), nil } // rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. func (f *Fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) keep1 := NewRow() // GTE keep2 := NewRow() // LTE // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit1 := (predicateMin >> uint(i)) & 1 bit2 := (predicateMax >> uint(i)) & 1 @@ -941,7 +941,7 @@ func (f *Fragment) top(opt TopOptions) ([]Pair, error) { // Calculate count and append. count := cnt if opt.Src != nil { - count = opt.Src.IntersectionCount(f.Row(rowID)) + count = opt.Src.IntersectionCount(f.row(rowID)) } if count == 0 { continue @@ -985,7 +985,7 @@ func (f *Fragment) top(opt TopOptions) ([]Pair, error) { // Calculate the intersecting column count and skip if it's below our // last row in our current result set. - count := opt.Src.IntersectionCount(f.Row(rowID)) + count := opt.Src.IntersectionCount(f.row(rowID)) if count < threshold { continue } @@ -1029,7 +1029,7 @@ func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair { continue } - row := f.Row(rowID) + row := f.row(rowID) if row.Count() > 0 { // Otherwise load from storage. pairs = append(pairs, BitmapPair{ @@ -1347,7 +1347,7 @@ func (f *Fragment) bulkImport(rowIDs, columnIDs []uint64) error { // Import should ALWAYS have row() load a new row from fragment.storage // because the row that's in rowCache hasn't been updated with // this import's data. - f.cache.BulkAdd(rowID, f.row(rowID, false, false).Count()) + f.cache.BulkAdd(rowID, f.unprotectedRow(rowID, false, false).Count()) } f.cache.Invalidate() diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 9038eea7d..e7a77dafb 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -49,18 +49,18 @@ func TestFragment_SetBit(t *testing.T) { } // Verify counts on rows. - if n := f.Row(120).Count(); n != 2 { + if n := f.row(120).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) - } else if n := f.Row(121).Count(); n != 1 { + } else if n := f.row(121).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(120).Count(); n != 2 { + } else if n := f.row(120).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) - } else if n := f.Row(121).Count(); n != 1 { + } else if n := f.row(121).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -80,14 +80,14 @@ func TestFragment_ClearBit(t *testing.T) { } // Verify count on row. - if n := f.Row(1000).Count(); n != 1 { + if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -539,14 +539,14 @@ func TestFragment_Snapshot(t *testing.T) { // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -1003,7 +1003,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Verify data in other fragment. - if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected columns: %+v", a) } @@ -1012,7 +1012,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { t.Fatal(err) } else if n := f1.cache.Len(); n != 1 { t.Fatalf("unexpected cache size (reopen): %d", n) - } else if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + } else if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected columns (reopen): %+v", a) } } @@ -1063,7 +1063,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Start benchmark b.ResetTimer() for i := 0; i < b.N; i++ { - if n := f.Row(1).IntersectionCount(f.Row(2)); n == 0 { + if n := f.row(1).IntersectionCount(f.row(2)); n == 0 { b.Fatalf("unexpected count: %d", n) } } @@ -1131,14 +1131,14 @@ func TestFragment_Snapshot_Run(t *testing.T) { // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 2 { + } else if n := f.row(1000).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 2 { + } else if n := f.row(1000).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) } } diff --git a/holder_test.go b/holder_test.go index 09fa47829..25d6b5105 100644 --- a/holder_test.go +++ b/holder_test.go @@ -430,25 +430,23 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0, 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 0, 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 0, 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 0, 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 0, 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 1, 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { + + if a := hldr.Row("y", "z", 3, 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -508,29 +506,23 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0, 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 0, 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 0, 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 0, 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 0, 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - a := f.Row(9).Columns() - if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a) - } - if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 1, 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + + if a := hldr.Row("y", "z", 2, 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -551,26 +543,24 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0, 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 0, 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 0, 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 0, 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 0, 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) + f := hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) if f != nil { t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) } - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 2, 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } diff --git a/test/holder.go b/test/holder.go index 401a9d36e..59c024afc 100644 --- a/test/holder.go +++ b/test/holder.go @@ -125,6 +125,34 @@ func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, return &Fragment{Fragment: frag} } +// Row returns a Row for a given field. +func (h *Holder) Row(index, field string, slice, rowID uint64) *pilosa.Row { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + if err != nil { + panic(err) + } + row, err := f.Row(slice, rowID) + if err != nil { + panic(err) + } + return row +} + +// ViewRow returns a Row for a given field and view. +func (h *Holder) ViewRow(index, field, view string, slice, rowID uint64) *pilosa.Row { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + if err != nil { + panic(err) + } + row, err := f.ViewRow(view, slice, rowID) + if err != nil { + panic(err) + } + return row +} + // SetBit clears a bit on the given field. func (h *Holder) SetBit(index, field string, rowID, columnID uint64) { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) diff --git a/view.go b/view.go index 265db0857..181562995 100644 --- a/view.go +++ b/view.go @@ -303,6 +303,12 @@ func (v *View) DeleteFragment(slice uint64) error { return nil } +// row returns a row for a slice of the view. +func (v *View) row(slice, rowID uint64) *Row { + frag := v.Fragment(slice) + return frag.row(rowID) +} + // SetBit sets a bit within the view. func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth From 38ae5b19f724140731cfeed7df6a8e326390807f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 7 Jun 2018 15:20:18 -0500 Subject: [PATCH 56/78] remove commented code --- executor_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/executor_test.go b/executor_test.go index 81fc74c72..a94f7e2fe 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1134,7 +1134,6 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } // Verify that one column is set on both node's holder. - //if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 { if n := hldr.ViewRow("i", "f", "standard_2016", 0, 10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } From 173939813fc6952b20217a283f523b8d9628212a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 7 Jun 2018 17:14:46 -0500 Subject: [PATCH 57/78] remove slice argment from Field.Row() method --- client_test.go | 6 +++--- executor_test.go | 8 ++++---- field.go | 10 +++++----- holder_test.go | 40 ++++++++++++++++++++-------------------- test/holder.go | 8 ++++---- view.go | 14 +++++++++++--- 6 files changed, 47 insertions(+), 39 deletions(-) diff --git a/client_test.go b/client_test.go index 6456ef386..84037d374 100644 --- a/client_test.go +++ b/client_test.go @@ -212,7 +212,7 @@ func TestClient_Import(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. hldr.SetBit("i", "f", 1, 0) // set a bit so the view gets created. - hldr.Row("i", "f", 0, 0) + hldr.Row("i", "f", 0) s := test.NewServer() defer s.Close() @@ -231,10 +231,10 @@ func TestClient_Import(t *testing.T) { } // Verify data. - if a := hldr.Row("i", "f", 0, 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { t.Fatalf("unexpected columns: %+v", a) } - if a := hldr.Row("i", "f", 0, 200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { + if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { t.Fatalf("unexpected columns: %+v", a) } } diff --git a/executor_test.go b/executor_test.go index a94f7e2fe..221a554e8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -241,7 +241,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { hldr.SetBit("i", "f", 1, 0) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if n := hldr.Row("i", "f", 0, 11).Count(); n != 0 { + if n := hldr.Row("i", "f", 11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } @@ -253,7 +253,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } } - if n := hldr.Row("i", "f", 0, 11).Count(); n != 1 { + if n := hldr.Row("i", "f", 11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { @@ -1080,7 +1080,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } // Verify that one column is set on both node's holder. - if n := hldr.Row("i", "f", 0, 10).Count(); n != 1 { + if n := hldr.Row("i", "f", 10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -1134,7 +1134,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } // Verify that one column is set on both node's holder. - if n := hldr.ViewRow("i", "f", "standard_2016", 0, 10).Count(); n != 1 { + if n := hldr.ViewRow("i", "f", "standard_2016", 10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { diff --git a/field.go b/field.go index 132458652..8d35de8fa 100644 --- a/field.go +++ b/field.go @@ -620,8 +620,8 @@ func (f *Field) DeleteView(name string) error { return nil } -// Row returns a row for a slice of the standard view. -func (f *Field) Row(slice, rowID uint64) (*Row, error) { +// Row returns a row of the standard view. +func (f *Field) Row(rowID uint64) (*Row, error) { if f.Type() != FieldTypeSet { return nil, errors.Errorf("row method unsupported for field type: %s", f.Type()) } @@ -629,17 +629,17 @@ func (f *Field) Row(slice, rowID uint64) (*Row, error) { if view == nil { return nil, ErrInvalidView } - return view.row(slice, rowID), nil + return view.row(rowID), nil } // ViewRow returns a row for a view and slice. // TODO: unexport this with views (it's only used in tests). -func (f *Field) ViewRow(viewName string, slice, rowID uint64) (*Row, error) { +func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) { view := f.View(viewName) if view == nil { return nil, ErrInvalidView } - return view.row(slice, rowID), nil + return view.row(rowID), nil } // SetBit sets a bit on a view within the field. diff --git a/holder_test.go b/holder_test.go index 25d6b5105..9739de65d 100644 --- a/holder_test.go +++ b/holder_test.go @@ -430,23 +430,23 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { - if a := hldr.Row("i", "f", 0, 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - if a := hldr.Row("i", "f0", 1, 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - if a := hldr.Row("y", "z", 3, 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -506,23 +506,23 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { - if a := hldr.Row("i", "f", 0, 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - if a := hldr.Row("i", "f0", 1, 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - if a := hldr.Row("y", "z", 2, 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -543,15 +543,15 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { - if a := hldr.Row("i", "f", 0, 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := hldr.Row("i", "f", 0, 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } @@ -560,7 +560,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) } - if a := hldr.Row("y", "z", 2, 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } diff --git a/test/holder.go b/test/holder.go index 59c024afc..f49613fb0 100644 --- a/test/holder.go +++ b/test/holder.go @@ -126,13 +126,13 @@ func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, } // Row returns a Row for a given field. -func (h *Holder) Row(index, field string, slice, rowID uint64) *pilosa.Row { +func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) if err != nil { panic(err) } - row, err := f.Row(slice, rowID) + row, err := f.Row(rowID) if err != nil { panic(err) } @@ -140,13 +140,13 @@ func (h *Holder) Row(index, field string, slice, rowID uint64) *pilosa.Row { } // ViewRow returns a Row for a given field and view. -func (h *Holder) ViewRow(index, field, view string, slice, rowID uint64) *pilosa.Row { +func (h *Holder) ViewRow(index, field, view string, rowID uint64) *pilosa.Row { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) if err != nil { panic(err) } - row, err := f.ViewRow(view, slice, rowID) + row, err := f.ViewRow(view, rowID) if err != nil { panic(err) } diff --git a/view.go b/view.go index 181562995..0049edaa1 100644 --- a/view.go +++ b/view.go @@ -304,9 +304,17 @@ func (v *View) DeleteFragment(slice uint64) error { } // row returns a row for a slice of the view. -func (v *View) row(slice, rowID uint64) *Row { - frag := v.Fragment(slice) - return frag.row(rowID) +func (v *View) row(rowID uint64) *Row { + row := NewRow() + for _, frag := range v.Fragments() { + fr := frag.row(rowID) + if fr == nil { + continue + } + row.Merge(fr) + } + return row + } // SetBit sets a bit within the view. From e58d40718209fbce63ae2e665fcadbccab52373e Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 7 Jun 2018 22:50:21 -0500 Subject: [PATCH 58/78] unexport (most) View methods --- cluster.go | 2 +- field.go | 32 ++++++------ holder.go | 8 +-- holder_test.go | 13 ++--- test/holder.go | 18 ------- view.go | 68 ++++++++++--------------- view_internal_test.go | 68 +++++++++++++++++++++++++ view_test.go | 115 ------------------------------------------ 8 files changed, 122 insertions(+), 202 deletions(-) create mode 100644 view_internal_test.go delete mode 100644 view_test.go diff --git a/cluster.go b/cluster.go index 9723e5bce..9cf54df08 100644 --- a/cluster.go +++ b/cluster.go @@ -629,7 +629,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost { for _, field := range idx.Fields() { for _, view := range field.Views() { - fieldViews.addView(field.Name(), view.Name()) + fieldViews.addView(field.Name(), view.name) } } diff --git a/field.go b/field.go index 8d35de8fa..8c7683cd1 100644 --- a/field.go +++ b/field.go @@ -137,7 +137,7 @@ func (f *Field) MaxSlice() uint64 { var max uint64 for _, view := range f.views { - if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max { + if viewMaxSlice := view.calculateMaxSlice(); viewMaxSlice > max { max = viewMaxSlice } } @@ -249,11 +249,11 @@ func (f *Field) openViews() error { name := filepath.Base(fi.Name()) view := f.newView(f.ViewPath(name), name) - if err := view.Open(); err != nil { - return fmt.Errorf("opening view: view=%s, err=%s", view.Name(), err) + if err := view.open(); err != nil { + return fmt.Errorf("opening view: view=%s, err=%s", view.name, err) } view.RowAttrStore = f.rowAttrStore - f.views[view.Name()] = view + f.views[view.name] = view } return nil @@ -369,7 +369,7 @@ func (f *Field) Close() error { // Close all views. for _, view := range f.views { - if err := view.Close(); err != nil { + if err := view.close(); err != nil { return err } } @@ -447,9 +447,9 @@ func (f *Field) deleteBSIGroupAndView(name string) error { if view := f.views[viewName]; view != nil { delete(f.views, viewName) - if err := view.Close(); err != nil { + if err := view.close(); err != nil { return errors.Wrap(err, "closing view") - } else if err := os.RemoveAll(view.Path()); err != nil { + } else if err := os.RemoveAll(view.path); err != nil { return errors.Wrap(err, "deleting directory") } } @@ -538,7 +538,7 @@ func (f *Field) viewNames() []string { // RecalculateCaches recalculates caches on every view in the field. func (f *Field) RecalculateCaches() { for _, view := range f.Views() { - view.RecalculateCaches() + view.recalculateCaches() } } @@ -579,11 +579,11 @@ func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) { view := f.newView(f.ViewPath(name), name) - if err := view.Open(); err != nil { + if err := view.open(); err != nil { return nil, false, errors.Wrap(err, "opening view") } view.RowAttrStore = f.rowAttrStore - f.views[view.Name()] = view + f.views[view.name] = view return view, true, nil } @@ -606,12 +606,12 @@ func (f *Field) DeleteView(name string) error { } // Close data files before deletion. - if err := view.Close(); err != nil { + if err := view.close(); err != nil { return errors.Wrap(err, "closing view") } // Delete view directory. - if err := os.RemoveAll(view.Path()); err != nil { + if err := os.RemoveAll(view.path); err != nil { return errors.Wrap(err, "deleting directory") } @@ -656,7 +656,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed } // Set non-time bit. - if v, err := view.SetBit(rowID, colID); err != nil { + if v, err := view.setBit(rowID, colID); err != nil { return changed, errors.Wrap(err, "setting on view") } else if v { changed = v @@ -674,7 +674,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed return changed, errors.Wrapf(err, "creating view %s", subname) } - if c, err := view.SetBit(rowID, colID); err != nil { + if c, err := view.setBit(rowID, colID); err != nil { return changed, errors.Wrapf(err, "setting on view %s", subname) } else if c { changed = true @@ -698,7 +698,7 @@ func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (change } // Clear non-time bit. - if v, err := view.ClearBit(rowID, colID); err != nil { + if v, err := view.clearBit(rowID, colID); err != nil { return changed, errors.Wrap(err, "clearing on view") } else if v { changed = v @@ -716,7 +716,7 @@ func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (change return changed, errors.Wrapf(err, "creating view %s", subname) } - if c, err := view.ClearBit(rowID, colID); err != nil { + if c, err := view.clearBit(rowID, colID); err != nil { return changed, errors.Wrapf(err, "clearing on view %s", subname) } else if c { changed = true diff --git a/holder.go b/holder.go index 89bd508ac..c7ae15e0d 100644 --- a/holder.go +++ b/holder.go @@ -217,7 +217,7 @@ func (h *Holder) Schema() []*IndexInfo { for _, field := range index.Fields() { fi := &FieldInfo{Name: field.Name(), Options: field.Options()} for _, view := range field.Views() { - fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) + fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } sort.Sort(viewInfoSlice(fi.Views)) di.Fields = append(di.Fields, fi) @@ -437,7 +437,7 @@ func (h *Holder) flushCaches() { for _, index := range h.Indexes() { for _, field := range index.Fields() { for _, view := range field.Views() { - for _, fragment := range view.Fragments() { + for _, fragment := range view.allFragments() { select { case <-h.closing: return @@ -808,14 +808,14 @@ func (c *HolderCleaner) CleanHolder() error { // Get the fragments registered in memory. for _, field := range index.Fields() { for _, view := range field.Views() { - for _, fragment := range view.Fragments() { + for _, fragment := range view.allFragments() { fragSlice := fragment.slice // Ignore fragments that should be present. if uint64InSlice(fragSlice, containedSlices) { continue } // Delete fragment. - if err := view.DeleteFragment(fragSlice); err != nil { + if err := view.deleteFragment(fragSlice); err != nil { return errors.Wrap(err, "deleting fragment") } } diff --git a/holder_test.go b/holder_test.go index 9739de65d..b5a494028 100644 --- a/holder_test.go +++ b/holder_test.go @@ -210,9 +210,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { + } else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -233,9 +231,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { + } else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -261,7 +257,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { + } else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } else if err := view.Fragment(0).FlushCache(); err != nil { t.Fatal(err) @@ -400,7 +396,8 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr0.SetBit("i", "f0", 9, SliceWidth+5) - hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0) + // Set a bit to create the fragment. + hldr0.SetBit("y", "z", 0, 0) // Set data on the remote holder. hldr1.SetBit("i", "f", 0, 4000) diff --git a/test/holder.go b/test/holder.go index f49613fb0..4484850fd 100644 --- a/test/holder.go +++ b/test/holder.go @@ -89,24 +89,6 @@ func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field { return f } -// MustCreateFragmentIfNotExists returns a given fragment. Panic on error. -func (h *Holder) MustCreateFragmentIfNotExists(index, field, view string, slice uint64) *Fragment { - idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) - if err != nil { - panic(err) - } - v, err := f.CreateViewIfNotExists(view) - if err != nil { - panic(err) - } - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - panic(err) - } - return &Fragment{Fragment: frag} -} - // MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, slice uint64) *Fragment { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) diff --git a/view.go b/view.go index 0049edaa1..a7aaf0cb8 100644 --- a/view.go +++ b/view.go @@ -82,20 +82,8 @@ func NewView(path, index, field, name string, cacheSize uint32) *View { } } -// Name returns the name the view was initialized with. -func (v *View) Name() string { return v.name } - -// Index returns the index name the view was initialized with. -func (v *View) Index() string { return v.index } - -// Field returns the field name the view was initialized with. -func (v *View) Field() string { return v.field } - -// Path returns the path the view was initialized with. -func (v *View) Path() string { return v.path } - -// Open opens and initializes the view. -func (v *View) Open() error { +// open opens and initializes the view. +func (v *View) open() error { // Never keep a cache for field views. if strings.HasPrefix(v.name, viewBSIGroupPrefix) { @@ -116,7 +104,7 @@ func (v *View) Open() error { return nil }(); err != nil { - v.Close() + v.close() return err } @@ -149,7 +137,7 @@ func (v *View) openFragments() error { continue } - frag := v.newFragment(v.FragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(slice), slice) if err := frag.Open(); err != nil { return fmt.Errorf("open fragment: slice=%d, err=%s", frag.slice, err) } @@ -160,8 +148,8 @@ func (v *View) openFragments() error { return nil } -// Close closes the view and its fragments. -func (v *View) Close() error { +// close closes the view and its fragments. +func (v *View) close() error { v.mu.Lock() defer v.mu.Unlock() @@ -176,8 +164,8 @@ func (v *View) Close() error { return nil } -// MaxSlice returns the max slice in the view. -func (v *View) MaxSlice() uint64 { +// calculateMaxSlice returns the max slice in the view. +func (v *View) calculateMaxSlice() uint64 { v.mu.RLock() defer v.mu.RUnlock() @@ -191,8 +179,8 @@ func (v *View) MaxSlice() uint64 { return max } -// FragmentPath returns the path to a fragment in the view. -func (v *View) FragmentPath(slice uint64) string { +// fragmentPath returns the path to a fragment in the view. +func (v *View) fragmentPath(slice uint64) string { return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10)) } @@ -205,8 +193,8 @@ func (v *View) Fragment(slice uint64) *Fragment { func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] } -// Fragments returns a list of all fragments in the view. -func (v *View) Fragments() []*Fragment { +// allFragments returns a list of all fragments in the view. +func (v *View) allFragments() []*Fragment { v.mu.Lock() defer v.mu.Unlock() @@ -217,9 +205,9 @@ func (v *View) Fragments() []*Fragment { return other } -// RecalculateCaches recalculates the cache on every fragment in the view. -func (v *View) RecalculateCaches() { - for _, fragment := range v.Fragments() { +// recalculateCaches recalculates the cache on every fragment in the view. +func (v *View) recalculateCaches() { + for _, fragment := range v.allFragments() { fragment.RecalculateCache() } } @@ -238,7 +226,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } // Initialize and open fragment. - frag := v.newFragment(v.FragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(slice), slice) if err := frag.Open(); err != nil { return nil, errors.Wrap(err, "opening fragment") } @@ -273,8 +261,8 @@ func (v *View) newFragment(path string, slice uint64) *Fragment { return frag } -// DeleteFragment removes the fragment from the view. -func (v *View) DeleteFragment(slice uint64) error { +// deleteFragment removes the fragment from the view. +func (v *View) deleteFragment(slice uint64) error { fragment := v.fragments[slice] if fragment == nil { @@ -306,7 +294,7 @@ func (v *View) DeleteFragment(slice uint64) error { // row returns a row for a slice of the view. func (v *View) row(rowID uint64) *Row { row := NewRow() - for _, frag := range v.Fragments() { + for _, frag := range v.allFragments() { fr := frag.row(rowID) if fr == nil { continue @@ -317,8 +305,8 @@ func (v *View) row(rowID uint64) *Row { } -// SetBit sets a bit within the view. -func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { +// setBit sets a bit within the view. +func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { @@ -327,8 +315,8 @@ func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { return frag.setBit(rowID, columnID) } -// ClearBit clears a bit within the view. -func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { +// clearBit clears a bit within the view. +func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { @@ -359,7 +347,7 @@ func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed b // sum returns the sum & count of a field. func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { - for _, f := range v.Fragments() { + for _, f := range v.allFragments() { fsum, fcount, err := f.sum(filter, bitDepth) if err != nil { return sum, count, err @@ -373,7 +361,7 @@ func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { // min returns the min and count of a field. func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) { var minHasValue bool - for _, f := range v.Fragments() { + for _, f := range v.allFragments() { fmin, fcount, err := f.min(filter, bitDepth) if err != nil { return min, count, err @@ -400,7 +388,7 @@ func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) { // max returns the max and count of a field. func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) { - for _, f := range v.Fragments() { + for _, f := range v.allFragments() { fmax, fcount, err := f.max(filter, bitDepth) if err != nil { return max, count, err @@ -416,7 +404,7 @@ func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) { // rangeOp returns rows with a field value encoding matching the predicate. func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { r := NewRow() - for _, frag := range v.Fragments() { + for _, frag := range v.allFragments() { other, err := frag.rangeOp(op, bitDepth, predicate) if err != nil { return nil, err @@ -430,7 +418,7 @@ func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, err // value between predicateMin and predicateMax. func (v *View) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { r := NewRow() - for _, frag := range v.Fragments() { + for _, frag := range v.allFragments() { other, err := frag.rangeBetween(bitDepth, predicateMin, predicateMax) if err != nil { return nil, err diff --git a/view_internal_test.go b/view_internal_test.go new file mode 100644 index 000000000..d0e8bfdd1 --- /dev/null +++ b/view_internal_test.go @@ -0,0 +1,68 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io/ioutil" + "testing" +) + +// mustOpenView returns a new instance of View with a temporary path. +func mustOpenView(index, field, name string) *View { + path, err := ioutil.TempDir("", "pilosa-view-") + if err != nil { + panic(err) + } + + v := NewView(path, index, field, name, DefaultCacheSize) + if err := v.open(); err != nil { + panic(err) + } + v.RowAttrStore = newMemAttrStore() + return v +} + +// Ensure view can open and retrieve a fragment. +func TestView_DeleteFragment(t *testing.T) { + v := mustOpenView("i", "f", "v") + defer v.close() + + slice := uint64(9) + + // Create fragment. + fragment, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } else if fragment == nil { + t.Fatal("expected fragment") + } + + err = v.deleteFragment(slice) + if err != nil { + t.Fatal(err) + } + + if v.Fragment(slice) != nil { + t.Fatal("fragment still exists in view") + } + + // Recreate fragment with same slice, verify that the old fragment was not reused. + fragment2, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } else if fragment == fragment2 { + t.Fatal("failed to create new fragment") + } +} diff --git a/view_test.go b/view_test.go deleted file mode 100644 index 04921786d..000000000 --- a/view_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" -) - -// View is a test wrapper for pilosa.View. -type View struct { - *pilosa.View - RowAttrStore pilosa.AttrStore -} - -// NewView returns a new instance of View with a temporary path. -func NewView(index, field, name string) *View { - path, err := ioutil.TempDir("", "pilosa-view-") - if err != nil { - panic(err) - } - - v := &View{ - View: pilosa.NewView(path, index, field, name, pilosa.DefaultCacheSize), - RowAttrStore: test.MustOpenAttrStore(), - } - v.View.RowAttrStore = v.RowAttrStore - return v -} - -// MustOpenView creates and opens an view at a temporary path. Panic on error. -func MustOpenView(index, field, name string) *View { - v := NewView(index, field, name) - if err := v.Open(); err != nil { - panic(err) - } - return v -} - -// Close closes the view and removes all underlying data. -func (v *View) Close() error { - defer os.Remove(v.Path()) - defer v.RowAttrStore.Close() - return v.View.Close() -} - -// Reopen closes the view and reopens it as a new instance. -func (v *View) Reopen() error { - path := v.Path() - if err := v.View.Close(); err != nil { - return err - } - - v.View = pilosa.NewView(path, v.Index(), v.Field(), v.Name(), pilosa.DefaultCacheSize) - v.View.RowAttrStore = v.RowAttrStore - return v.Open() -} - -// MustClearColumns clears columns on a row. Panic on error. -func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := v.ClearBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// Ensure view can open and retrieve a fragment. -func TestView_DeleteFragment(t *testing.T) { - v := MustOpenView("i", "f", "v") - defer v.Close() - - slice := uint64(9) - - // Create fragment. - fragment, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - t.Fatal(err) - } else if fragment == nil { - t.Fatal("expected fragment") - } - - err = v.DeleteFragment(slice) - if err != nil { - t.Fatal(err) - } - - if v.Fragment(slice) != nil { - t.Fatal("fragment still exists in view") - } - - // Recreate fragment with same slice, verify that the old fragment was not reused. - fragment2, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - t.Fatal(err) - } else if fragment == fragment2 { - t.Fatal("failed to create new fragment") - } -} From f2c104dfef75d30854c235fa9dc5c78905024651 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 12 Jun 2018 13:22:40 -0500 Subject: [PATCH 59/78] Migrate HTTP handler and client into http subpackage. --- cache.go | 2 +- client.go | 1110 ++------------- cluster.go | 9 +- ctl/common.go | 6 +- ctl/import.go | 9 +- executor.go | 31 +- fragment.go | 15 +- handler.go | 1187 +--------------- holder.go | 8 +- holder_test.go | 27 +- http/client.go | 1034 ++++++++++++++ client_test.go => http/client_test.go | 27 +- http/handler.go | 1231 +++++++++++++++++ .../handler_internal_test.go | 10 +- handler_test.go => http/handler_test.go | 101 +- pilosa.go | 10 +- row.go | 8 +- server.go | 45 +- server/server.go | 39 +- server/server_test.go | 3 +- test/client.go | 10 +- test/executor.go | 11 +- test/handler.go | 15 +- test/pilosa.go | 16 +- 24 files changed, 2577 insertions(+), 2387 deletions(-) create mode 100644 http/client.go rename client_test.go => http/client_test.go (93%) create mode 100644 http/handler.go rename handler_internal_test.go => http/handler_internal_test.go (94%) rename handler_test.go => http/handler_test.go (93%) diff --git a/cache.go b/cache.go index 06046220a..ec9ade91b 100644 --- a/cache.go +++ b/cache.go @@ -409,7 +409,7 @@ func (p Pairs) String() string { return buf.String() } -func encodePairs(a Pairs) []*internal.Pair { +func EncodePairs(a Pairs) []*internal.Pair { other := make([]*internal.Pair, len(a)) for i := range a { other[i] = encodePair(a[i]) diff --git a/client.go b/client.go index 81f750a39..56cba08e1 100644 --- a/client.go +++ b/client.go @@ -1,878 +1,13 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package pilosa import ( - "bytes" "context" - "encoding/json" - "fmt" "io" - "io/ioutil" - "math/rand" - "net/http" - "net/url" - "sort" - "strconv" - - "crypto/tls" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" - "github.com/pkg/errors" ) -// ClientOptions represents the configuration for a InternalHTTPClient -type ClientOptions struct { - TLS *tls.Config -} - -// InternalHTTPClient represents a client to the Pilosa cluster. -type InternalHTTPClient struct { - defaultURI *URI - - // The client to use for HTTP communication. - HTTPClient *http.Client -} - -// NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host. -func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTTPClient, error) { - if host == "" { - return nil, ErrHostRequired - } - - uri, err := NewURIFromAddress(host) - if err != nil { - return nil, errors.Wrap(err, "getting URI") - } - - client := NewInternalHTTPClientFromURI(uri, remoteClient) - return client, nil -} - -func NewInternalHTTPClientFromURI(defaultURI *URI, remoteClient *http.Client) *InternalHTTPClient { - return &InternalHTTPClient{ - defaultURI: defaultURI, - HTTPClient: remoteClient, - } -} - -// Host returns the host the client was initialized with. -func (c *InternalHTTPClient) Host() *URI { return c.defaultURI } - -// MaxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByIndex(ctx) -} - -// maxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/slices/max") - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var rsp getSlicesMaxResponse - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - - return rsp.Standard, nil -} - -// Schema returns all index and field schema information. -func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) { - // Execute request against the host. - u := c.defaultURI.Path("/schema") - - // Build request. - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var rsp getSchemaResponse - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - return rsp.Indexes, nil -} - -// CreateIndex creates a new index on the server. -func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { - // Encode query request. - buf, err := json.Marshal(&postIndexRequest{ - Options: opt, - }) - if err != nil { - return errors.Wrap(err, "encoding request") - } - - // Create URL & HTTP request. - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index)) - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } - - // Handle response based on status code. - switch resp.StatusCode { - case http.StatusOK: - return nil // ok - case http.StatusConflict: - return ErrIndexExists - default: - return errors.New(string(body)) - } -} - -// FragmentNodes returns a list of nodes that own a slice. -func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { - // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/fragment/nodes") - u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var a []*Node - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&a); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - - return a, nil -} - -// Query executes query against the index. -func (c *InternalHTTPClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { - return c.QueryNode(ctx, c.defaultURI, index, queryRequest) -} - -// QueryNode executes query against the index, sending the request to the node specified. -func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { - if index == "" { - return nil, ErrIndexRequired - } else if queryRequest.Query == "" { - return nil, ErrQueryRequired - } - - // Encode request object. - buf, err := proto.Marshal(queryRequest) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Create HTTP request. - u := uri.Path(fmt.Sprintf("/index/%s/query", index)) - req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return nil, errors.New(string(body)) - } - - qresp := &internal.QueryResponse{} - if err := proto.Unmarshal(body, qresp); err != nil { - return nil, fmt.Errorf("unmarshal response: %s", err) - } else if s := qresp.Err; s != "" { - return nil, errors.New(s) - } - - return qresp, nil -} - -// Import bulk imports bits for a single slice to a host. -func (c *InternalHTTPClient) Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error { - if index == "" { - return ErrIndexRequired - } else if field == "" { - return ErrFieldRequired - } - - buf, err := marshalImportPayload(index, field, slice, bits) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Import to each node. - for _, node := range nodes { - if err := c.importNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - } - - return nil -} - -// ImportK bulk imports bits specified by string keys to a host. -func (c *InternalHTTPClient) ImportK(ctx context.Context, index, field string, columns []Bit) error { - if index == "" { - return ErrIndexRequired - } else if field == "" { - return ErrFieldRequired - } - - buf, err := marshalImportPayloadK(index, field, columns) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - node := &Node{ - URI: *c.defaultURI, - } - - // Import to node. - if err := c.importNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - - return nil -} - -func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { - err := c.CreateIndex(ctx, name, options) - if err == nil || err == ErrIndexExists { - return nil - } - return err -} - -func (c *InternalHTTPClient) EnsureField(ctx context.Context, indexName string, fieldName string, options FieldOptions) error { - err := c.CreateField(ctx, indexName, fieldName, options) - if err == nil || err == ErrFieldExists { - return nil - } - return err -} - -// marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, field string, slice uint64, bits []Bit) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - rowIDs := Bits(bits).RowIDs() - columnIDs := Bits(bits).ColumnIDs() - timestamps := Bits(bits).Timestamps() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ - Index: index, - Field: field, - Slice: slice, - RowIDs: rowIDs, - ColumnIDs: columnIDs, - Timestamps: timestamps, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. -func marshalImportPayloadK(index, field string, bits []Bit) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - rowKeys := Bits(bits).RowKeys() - columnKeys := Bits(bits).ColumnKeys() - timestamps := Bits(bits).Timestamps() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ - Index: index, - Field: field, - RowKeys: rowKeys, - ColumnKeys: columnKeys, - Timestamps: timestamps, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// importNode sends a pre-marshaled import request to a node. -func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []byte) error { - // Create URL & HTTP request. - u := nodePathToURL(node, "/import") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return errors.New(string(body)) - } - - var isresp internal.ImportResponse - if err := proto.Unmarshal(body, &isresp); err != nil { - return fmt.Errorf("unmarshal import response: %s", err) - } else if s := isresp.Err; s != "" { - return errors.New(s) - } - - return nil -} - -// ImportValue bulk imports field values for a single slice to a host. -func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error { - if index == "" { - return ErrIndexRequired - } else if field == "" { - return ErrFieldRequired - } - - buf, err := marshalImportValuePayload(index, field, slice, vals) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Import to each node. - for _, node := range nodes { - if err := c.importValueNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - } - - return nil -} - -// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func marshalImportValuePayload(index, field string, slice uint64, vals []FieldValue) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - columnIDs := FieldValues(vals).ColumnIDs() - values := FieldValues(vals).Values() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportValueRequest{ - Index: index, - Field: field, - Slice: slice, - ColumnIDs: columnIDs, - Values: values, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// importValueNode sends a pre-marshaled import request to a node. -func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, buf []byte) error { - // Create URL & HTTP request. - u := nodePathToURL(node, "/import-value") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return errors.New(string(body)) - } - - var isresp internal.ImportResponse - if err := proto.Unmarshal(body, &isresp); err != nil { - return fmt.Errorf("unmarshal import response: %s", err) - } else if s := isresp.Err; s != "" { - return errors.New(s) - } - - return nil -} - -// ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { - if index == "" { - return ErrIndexRequired - } else if field == "" { - return ErrFieldRequired - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Attempt nodes in random order. - var e error - for _, i := range rand.Perm(len(nodes)) { - node := nodes[i] - - if err := c.exportNodeCSV(ctx, node, index, field, slice, w); err != nil { - e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) - continue - } else { - return nil - } - } - - return e -} - -// exportNode copies a CSV export from a node to w. -func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, field string, slice uint64, w io.Writer) error { - // Create URL. - u := nodePathToURL(node, "/export") - u.RawQuery = url.Values{ - "index": {index}, - "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Generate HTTP request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Accept", "text/csv") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Validate status code. - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("invalid status: %d", resp.StatusCode) - } - - // Copy body to writer. - if _, err := io.Copy(w, resp.Body); err != nil { - return errors.Wrap(err, "copying") - } - - return nil -} - -func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) { - node := &Node{ - URI: uri, - } - return c.backupSliceNode(ctx, index, field, slice, node) -} - -func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *Node) (io.ReadCloser, error) { - u := nodePathToURL(node, "/fragment/data") - u.RawQuery = url.Values{ - "index": {index}, - "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - - // Return error if status is not OK. - if resp.StatusCode == http.StatusNotFound { - resp.Body.Close() - return nil, ErrFragmentNotFound - } else if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode) - } - - return resp.Body, nil -} - -// CreateField creates a new field on the server. -func (c *InternalHTTPClient) CreateField(ctx context.Context, index, field string, opt FieldOptions) error { - if index == "" { - return ErrIndexRequired - } - - // Encode query request. - buf, err := json.Marshal(&postFieldRequest{ - Options: opt, - }) - if err != nil { - return errors.Wrap(err, "marshaling") - } - - // Create URL & HTTP request. - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s", index, field)) - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } - - // Handle response based on status code. - switch resp.StatusCode { - case http.StatusOK: - return nil // ok - case http.StatusConflict: - return ErrFieldExists - default: - return errors.New(string(body)) - } -} - -// FragmentBlocks returns a list of block checksums for a fragment on a host. -// Only returns blocks which contain data. -func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, field string, slice uint64) ([]FragmentBlock, error) { - u := uriPathToURL(c.defaultURI, "/fragment/blocks") - u.RawQuery = url.Values{ - "index": {index}, - "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - case http.StatusNotFound: - return nil, ErrFragmentNotFound - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp getFragmentBlocksResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Blocks, nil -} - -// BlockData returns row/column id pairs for a block. -func (c *InternalHTTPClient) BlockData(ctx context.Context, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { - buf, err := proto.Marshal(&internal.BlockDataRequest{ - Index: index, - Field: field, - Slice: slice, - Block: uint64(block), - }) - if err != nil { - return nil, nil, errors.Wrap(err, "marshaling") - } - - u := uriPathToURL(c.defaultURI, "/fragment/block/data") - req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/protobuf") - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Accept", "application/protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // fallthrough - case http.StatusNotFound: - return nil, nil, nil - default: - return nil, nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp internal.BlockDataResponse - if body, err := ioutil.ReadAll(resp.Body); err != nil { - return nil, nil, errors.Wrap(err, "reading") - } else if err := proto.Unmarshal(body, &rsp); err != nil { - return nil, nil, errors.Wrap(err, "unmarshalling") - } - return rsp.RowIDs, rsp.ColumnIDs, nil -} - -// ColumnAttrDiff returns data from differing blocks on a remote host. -func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/attr/diff", index)) - - // Encode request. - buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Build request. - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp postIndexAttrDiffResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Attrs, nil -} - -// RowAttrDiff returns data from differing blocks on a remote host. -func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s/attr/diff", index, field)) - - // Encode request. - buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks}) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Build request. - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - case http.StatusNotFound: - return nil, ErrFieldNotFound - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp postFieldAttrDiffResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Attrs, nil -} - -// SendMessage posts a message synchronously. -func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { - msg, err := MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshaling message: %v", err) - } - - u := uriPathToURL(uri, "/cluster/message") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) - if err != nil { - return errors.Wrap(err, "making new request") - } - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return fmt.Errorf("executing http request: %v", err) - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response body: %v", err) - } - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - default: - return fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body) - } - - return nil -} - // Bit represents the intersection of a row and a column. It can be specifed by // integer ids or string keys. type Bit struct { @@ -883,83 +18,6 @@ type Bit struct { Timestamp int64 } -// Bits is a slice of Bit. -type Bits []Bit - -func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p Bits) Len() int { return len(p) } - -func (p Bits) Less(i, j int) bool { - if p[i].RowID == p[j].RowID { - if p[i].ColumnID < p[j].ColumnID { - return p[i].Timestamp < p[j].Timestamp - } - return p[i].ColumnID < p[j].ColumnID - } - return p[i].RowID < p[j].RowID -} - -// RowIDs returns a slice of all the row IDs. -func (p Bits) RowIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].RowID - } - return other -} - -// ColumnIDs returns a slice of all the column IDs. -func (p Bits) ColumnIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].ColumnID - } - return other -} - -// RowKeys returns a slice of all the row keys. -func (p Bits) RowKeys() []string { - other := make([]string, len(p)) - for i := range p { - other[i] = p[i].RowKey - } - return other -} - -// ColumnKeys returns a slice of all the column keys. -func (p Bits) ColumnKeys() []string { - other := make([]string, len(p)) - for i := range p { - other[i] = p[i].ColumnKey - } - return other -} - -// Timestamps returns a slice of all the timestamps. -func (p Bits) Timestamps() []int64 { - other := make([]int64, len(p)) - for i := range p { - other[i] = p[i].Timestamp - } - return other -} - -// GroupBySlice returns a map of bits by slice. -func (p Bits) GroupBySlice() map[uint64][]Bit { - m := make(map[uint64][]Bit) - for _, bit := range p { - slice := bit.ColumnID / SliceWidth - m[slice] = append(m[slice], bit) - } - - for slice, bits := range m { - sort.Sort(Bits(bits)) - m[slice] = bits - } - - return m -} - // FieldValues represents the value for a column within a // range-encoded field. type FieldValue struct { @@ -967,79 +25,6 @@ type FieldValue struct { Value int64 } -// FieldValues represents a slice of field values. -type FieldValues []FieldValue - -func (p FieldValues) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p FieldValues) Len() int { return len(p) } - -func (p FieldValues) Less(i, j int) bool { - return p[i].ColumnID < p[j].ColumnID -} - -// ColumnIDs returns a slice of all the column IDs. -func (p FieldValues) ColumnIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].ColumnID - } - return other -} - -// Values returns a slice of all the values. -func (p FieldValues) Values() []int64 { - other := make([]int64, len(p)) - for i := range p { - other[i] = p[i].Value - } - return other -} - -// GroupBySlice returns a map of field values by slice. -func (p FieldValues) GroupBySlice() map[uint64][]FieldValue { - m := make(map[uint64][]FieldValue) - for _, val := range p { - slice := val.ColumnID / SliceWidth - m[slice] = append(m[slice], val) - } - - for slice, vals := range m { - sort.Sort(FieldValues(vals)) - m[slice] = vals - } - - return m -} - -// BitsByPos is a slice of bits sorted row then column. -type BitsByPos []Bit - -func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p BitsByPos) Len() int { return len(p) } -func (p BitsByPos) Less(i, j int) bool { - p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID) - if p0 == p1 { - return p[i].Timestamp < p[j].Timestamp - } - return p0 < p1 -} - -func uriPathToURL(uri *URI, path string) url.URL { - return url.URL{ - Scheme: uri.Scheme(), - Host: uri.HostPort(), - Path: path, - } -} - -func nodePathToURL(node *Node, path string) url.URL { - return url.URL{ - Scheme: node.URI.Scheme(), - Host: node.URI.HostPort(), - Path: path, - } -} - // InternalClient should be implemented by any struct that enables any transport between nodes // TODO: Refactor // Note from Travis: Typically an interface containing more than two or three methods is an indication that @@ -1060,9 +45,96 @@ type InternalClient interface { ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error CreateField(ctx context.Context, index, field string, opt FieldOptions) error - FragmentBlocks(ctx context.Context, index, field string, slice uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, index, field string, slice uint64, block int) ([]uint64, []uint64, error) - ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - RowAttrDiff(ctx context.Context, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error) + BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) + ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error + RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) +} + +//=============== + +type InternalQueryClient interface { + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) +} + +type NopInternalQueryClient struct{} + +func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return nil, nil +} + +func NewNopInternalQueryClient() *NopInternalQueryClient { + return &NopInternalQueryClient{} +} + +var _ InternalQueryClient = NewNopInternalQueryClient() + +//=============== + +type NopInternalClient struct{} + +func NewNopInternalClient() *NopInternalClient { + return &NopInternalClient{} +} + +var _ InternalClient = NewNopInternalClient() + +func (n *NopInternalClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { + return nil, nil +} +func (n *NopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { + return nil, nil +} +func (n *NopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { + return nil +} +func (n *NopInternalClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { + return nil, nil +} +func (n *NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return nil, nil +} +func (n *NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return nil, nil +} +func (n *NopInternalClient) Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error { + return nil +} +func (n *NopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { + return nil +} +func (n *NopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { + return nil +} +func (n *NopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string, options FieldOptions) error { + return nil +} +func (n *NopInternalClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error { + return nil +} +func (n *NopInternalClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { + return nil +} +func (n *NopInternalClient) CreateField(ctx context.Context, index, field string, opt FieldOptions) error { + return nil +} +func (n *NopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error) { + return nil, nil +} +func (n *NopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { + return nil, nil, nil +} +func (n *NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { + return nil, nil +} +func (n *NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { + return nil, nil +} +func (n *NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { + return nil +} +func (n *NopInternalClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) { + return nil, nil } diff --git a/cluster.go b/cluster.go index 9723e5bce..81ac03424 100644 --- a/cluster.go +++ b/cluster.go @@ -266,6 +266,8 @@ type Cluster struct { // RemoteClient *http.Client + + InternalClient InternalClient } // NewCluster returns a new instance of Cluster with defaults. @@ -281,6 +283,8 @@ func NewCluster() *Cluster { closing: make(chan struct{}), joining: make(chan struct{}), + InternalClient: NewNopInternalClient(), + Logger: NopLogger, } } @@ -1230,9 +1234,6 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err return errors.Wrap(err, "applying schema") } - // Create a client for calling remote nodes. - client := NewInternalHTTPClientFromURI(&c.Node.URI, c.RemoteClient) // TODO: ClientOptions - // Request each source file in ResizeSources. for _, src := range instr.Sources { c.Logger.Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) @@ -1259,7 +1260,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Stream slice from remote node. c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) - rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.Slice, srcURI) + rd, err := c.InternalClient.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.Slice, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found // on the remote node. This occurs when a slice has been skipped and diff --git a/ctl/common.go b/ctl/common.go index 11042a704..fe0c46f7b 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -17,7 +17,7 @@ package ctl import ( "crypto/tls" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pkg/errors" "github.com/spf13/pflag" @@ -37,7 +37,7 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP } // CommandClient returns a pilosa.InternalHTTPClient for the command -func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error) { +func CommandClient(cmd CommandWithTLSSupport) (*http.InternalHTTPClient, error) { tlsConfig := cmd.TLSConfiguration() var TLSConfig *tls.Config if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" { @@ -50,7 +50,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error InsecureSkipVerify: tlsConfig.SkipVerify, } } - client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig)) + client, err := http.NewInternalHTTPClient(cmd.TLSHost(), http.GetHTTPClient(TLSConfig)) if err != nil { return nil, errors.Wrap(err, "getting internal client") } diff --git a/ctl/import.go b/ctl/import.go index 9383c300f..c4d65f232 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -26,6 +26,7 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pkg/errors" ) @@ -245,12 +246,12 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err // Group bits by slice. logger.Printf("grouping %d bits", len(bits)) - bitsBySlice := pilosa.Bits(bits).GroupBySlice() + bitsBySlice := http.Bits(bits).GroupBySlice() // Parse path into bits. for slice, chunk := range bitsBySlice { if cmd.Sort { - sort.Sort(pilosa.BitsByPos(chunk)) + sort.Sort(http.BitsByPos(chunk)) } logger.Printf("importing slice: %d, n=%d", slice, len(chunk)) @@ -439,12 +440,12 @@ func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldV // Group vals by slice. logger.Printf("grouping %d vals", len(vals)) - valsBySlice := pilosa.FieldValues(vals).GroupBySlice() + valsBySlice := http.FieldValues(vals).GroupBySlice() // Parse path into FieldValues. for slice, vals := range valsBySlice { if cmd.Sort { - sort.Sort(pilosa.FieldValues(vals)) + sort.Sort(http.FieldValues(vals)) } logger.Printf("importing slice: %d, n=%d", slice, len(vals)) diff --git a/executor.go b/executor.go index 411039d40..4e2ffe9bf 100644 --- a/executor.go +++ b/executor.go @@ -17,7 +17,6 @@ package pilosa import ( "context" "fmt" - "net/http" "sort" "time" @@ -47,19 +46,35 @@ type Executor struct { Cluster *Cluster // Client used for remote requests. - client InternalClient + client InternalQueryClient // Maximum number of SetBit() or ClearBit() commands per request. MaxWritesPerRequest int } -// NewExecutor returns a new instance of Executor. -func NewExecutor(remoteClient *http.Client) *Executor { - return &Executor{ - client: NewInternalHTTPClientFromURI(nil, remoteClient), +type ExecutorOpt func(e *Executor) error + +func ExecutorOptInternalQueryClient(c InternalQueryClient) ExecutorOpt { + return func(e *Executor) error { + e.client = c + return nil } } +// NewExecutor returns a new instance of Executor. +func NewExecutor(opts ...ExecutorOpt) *Executor { + e := &Executor{ + client: NewNopInternalQueryClient(), + } + for _, opt := range opts { + err := opt(e) + if err != nil { + panic(err) + } + } + return e +} + // Execute executes a PQL query. func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { // Verify that an index is set. @@ -1380,7 +1395,7 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * case "SetRowAttrs": case "SetColumnAttrs": default: - v, err = decodeRow(pb.Results[i].GetRow()), nil + v, err = DecodeRow(pb.Results[i].GetRow()), nil } if err != nil { return nil, err @@ -1619,7 +1634,7 @@ func (vc *ValCount) Add(other ValCount) ValCount { } } -func encodeValCount(vc ValCount) *internal.ValCount { +func EncodeValCount(vc ValCount) *internal.ValCount { return &internal.ValCount{ Val: vc.Val, Count: vc.Count, diff --git a/fragment.go b/fragment.go index 1dfa9d09f..afa80f5f1 100644 --- a/fragment.go +++ b/fragment.go @@ -1782,8 +1782,7 @@ func (s *FragmentSyncer) SyncFragment() error { } // Retrieve remote blocks. - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Field(), s.Fragment.Slice()) + blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.Index(), s.Fragment.Field(), s.Fragment.Slice()) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } @@ -1847,7 +1846,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Read pairs from each remote block. var pairSets []PairSet - var clients []InternalClient + var uris []*URI for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) { if s.Node.ID == node.ID { continue @@ -1858,11 +1857,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { return nil } - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - clients = append(clients, client) + uri := &node.URI + uris = append(uris, uri) // Only sync the standard block. - rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Field(), f.Slice(), id) + rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.Index(), f.Field(), f.Slice(), id) if err != nil { return errors.Wrap(err, "getting block") } @@ -1885,7 +1884,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { } // Write updates to remote blocks. - for i := 0; i < len(clients); i++ { + for i := 0; i < len(uris); i++ { set, clear := sets[i], clears[i] count := 0 @@ -1924,7 +1923,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { Query: buffers[k].String(), Remote: true, } - _, err := clients[i].Query(context.Background(), f.Index(), queryRequest) + _, err := s.Cluster.InternalClient.QueryNode(context.Background(), uris[i], f.Index(), queryRequest) if err != nil { return errors.Wrap(err, "executing") } diff --git a/handler.go b/handler.go index c4bb364ea..e70368c49 100644 --- a/handler.go +++ b/handler.go @@ -1,947 +1,8 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package pilosa import ( "encoding/json" - "expvar" - "fmt" - "io" - "io/ioutil" "net/http" - "net/url" - // Imported for its side-effect of registering pprof endpoints with the server. - _ "net/http/pprof" - "reflect" - "runtime/debug" - "strconv" - "strings" - "time" - - "github.com/gogo/protobuf/proto" - "github.com/gorilla/handlers" - "github.com/gorilla/mux" - "github.com/pilosa/pilosa/internal" - - "github.com/pkg/errors" -) - -// Handler represents an HTTP handler. -type Handler struct { - Handler http.Handler - - Logger Logger - - // Keeps the query argument validators for each handler - validators map[string]*queryValidationSpec - - API *API - - AllowedOrigins []string -} - -// externalPrefixFlag denotes endpoints that are intended to be exposed to clients. -// This is used for stats tagging. -var externalPrefixFlag = map[string]bool{ - "schema": true, - "query": true, - "import": true, - "export": true, - "index": true, - "field": true, - "nodes": true, - "version": true, -} - -type errorResponse struct { - Error string `json:"error"` -} - -// HandlerOption is a functional option type for pilosa.Handler -type HandlerOption func(s *Handler) error - -func OptHandlerAllowedOrigins(origins []string) HandlerOption { - return func(h *Handler) error { - h.Handler = handlers.CORS( - handlers.AllowedOrigins(origins), - handlers.AllowedHeaders([]string{"Content-Type"}), - )(h.Handler) - return nil - } -} - -// NewHandler returns a new instance of Handler with a default logger. -func NewHandler(opts ...HandlerOption) (*Handler, error) { - handler := &Handler{ - Logger: NopLogger, - } - handler.Handler = NewRouter(handler) - handler.populateValidators() - - for _, opt := range opts { - err := opt(handler) - if err != nil { - return nil, errors.Wrap(err, "applying option") - } - } - - return handler, nil -} - -func (h *Handler) populateValidators() { - h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") - h.validators["GetSliceMax"] = queryValidationSpecRequired() - h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") - h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "slice") -} - -func (h *Handler) queryArgValidator(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - key := mux.CurrentRoute(r).GetName() - if validator, ok := h.validators[key]; ok { - if err := validator.validate(r.URL.Query()); err != nil { - // TODO: Return the response depending on the Accept header - response := errorResponse{Error: err.Error()} - body, err := json.Marshal(response) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - http.Error(w, string(body), http.StatusBadRequest) - return - } - } - next.ServeHTTP(w, r) - }) -} - -// NewRouter creates a new mux http router. -func NewRouter(handler *Handler) *mux.Router { - router := mux.NewRouter() - router.HandleFunc("/", handler.handleHome).Methods("GET") - router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") - router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") - router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.Handle("/debug/vars", expvar.Handler()).Methods("GET") - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") - router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") - router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") - - router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") - - router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") - router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.Handle("/debug/vars", expvar.Handler()).Methods("GET") - router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") - router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") - router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") - router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") - router.HandleFunc("/import", handler.handlePostImport).Methods("POST") - router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") - router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") - router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") - router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") - router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") - router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") - //router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented. - router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") - router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") - router.HandleFunc("/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST") - router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") - router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") - - // TODO: Apply MethodNotAllowed statuses to all endpoints. - // Ideally this would be automatic, as described in this (wontfix) ticket: - // https://github.com/gorilla/mux/issues/6 - // For now we just do it for the most commonly used handler, /query - router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") - - router.Use(handler.queryArgValidator) - return router -} - -func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) -} - -// ServeHTTP handles an HTTP request. -func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - defer func() { - if err := recover(); err != nil { - w.WriteHeader(http.StatusInternalServerError) - stack := debug.Stack() - msg := "PANIC: %s\n%s" - h.Logger.Printf(msg, err, stack) - fmt.Fprintf(w, msg, err, stack) - } - }() - - t := time.Now() - h.Handler.ServeHTTP(w, r) - dif := time.Since(t) - - // Calculate per request StatsD metrics when the handler is fully configured. - statsTags := make([]string, 0, 3) - - longQueryTime := h.API.LongQueryTime() - if longQueryTime > 0 && dif > longQueryTime { - h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) - statsTags = append(statsTags, "slow_query") - } - - pathParts := strings.Split(r.URL.Path, "/") - endpointName := strings.Join(pathParts, "_") - - if externalPrefixFlag[pathParts[1]] { - statsTags = append(statsTags, "external") - } - - // useragent tag identifies internal/external endpoints - statsTags = append(statsTags, "useragent:"+r.UserAgent()) - stats := h.API.StatsWithTags(statsTags) - if stats != nil { - stats.Histogram("http."+endpointName, float64(dif), 0.1) - } -} - -func (h *Handler) handleHome(w http.ResponseWriter, r *http.Request) { - http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.", http.StatusNotFound) -} - -// handleGetSchema handles GET /schema requests. -func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { - schema := h.API.Schema(r.Context()) - if err := json.NewEncoder(w).Encode(getSchemaResponse{ - Indexes: schema, - }); err != nil { - h.Logger.Printf("write schema response error: %s", err) - } -} - -// handleGetStatus handles GET /status requests. -func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - status := getStatusResponse{ - State: h.API.State(), - Nodes: h.API.Hosts(r.Context()), - LocalID: h.API.LocalID(), - } - if err := json.NewEncoder(w).Encode(status); err != nil { - h.Logger.Printf("write status response error: %s", err) - } -} - -func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { - info := h.API.Info() - if err := json.NewEncoder(w).Encode(info); err != nil { - h.Logger.Printf("write info response error: %s", err) - } -} - -type getSchemaResponse struct { - Indexes []*IndexInfo `json:"indexes"` -} - -type getStatusResponse struct { - State string `json:"state"` - Nodes []*Node `json:"nodes"` - LocalID string `json:"localID"` -} - -// handlePostQuery handles /query requests. -func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { - // Parse incoming request. - req, err := h.readQueryRequest(r) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - // TODO: Remove - req.Index = mux.Vars(r)["index"] - - resp, err := h.API.Query(r.Context(), req) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - - // Set appropriate status code, if there is an error. - if resp.Err != nil { - switch resp.Err { - case ErrTooManyWrites: - w.WriteHeader(http.StatusRequestEntityTooLarge) - default: - w.WriteHeader(http.StatusInternalServerError) - } - } - - // Write response back to client. - if err := h.writeQueryResponse(w, r, &resp); err != nil { - h.Logger.Printf("write query response error: %s", err) - } -} - -// handleGetSlicesMax handles GET /schema requests. -func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { - if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ - Standard: h.API.MaxSlices(r.Context()), - }); err != nil { - h.Logger.Printf("write slices-max response error: %s", err) - } -} - -type getSlicesMaxResponse struct { - Standard map[string]uint64 `json:"standard"` -} - -// handleGetIndexes handles GET /index request. -func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { - h.handleGetSchema(w, r) -} - -// handleGetIndex handles GET /index/ requests. -func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - index, err := h.API.Index(r.Context(), indexName) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - if err := json.NewEncoder(w).Encode(getIndexResponse{ - map[string]string{"name": index.Name()}, - }); err != nil { - h.Logger.Printf("write response error: %s", err) - } -} - -type getIndexResponse struct { - Index map[string]string `json:"index"` -} - -type postIndexRequest struct { - Options IndexOptions `json:"options"` -} - -//_postIndexRequest is necessary to avoid recursion while decoding. -type _postIndexRequest postIndexRequest - -// Custom Unmarshal JSON to validate request body when creating a new index. -func (p *postIndexRequest) UnmarshalJSON(b []byte) error { - - // m is an overflow map used to capture additional, unexpected keys. - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return errors.Wrap(err, "unmarshalling unexpected values") - } - - validIndexOptions := getValidOptions(IndexOptions{}) - err := validateOptions(m, validIndexOptions) - if err != nil { - return err - } - // Unmarshal expected values. - var _p _postIndexRequest - if err := json.Unmarshal(b, &_p); err != nil { - return errors.Wrap(err, "unmarshalling expected values") - } - - p.Options = _p.Options - - return nil -} - -// Raise errors for any unknown key -func validateOptions(data map[string]interface{}, validIndexOptions []string) error { - for k, v := range data { - switch k { - case "options": - options, ok := v.(map[string]interface{}) - if !ok { - return errors.New("options is not map[string]interface{}") - } - for kk, vv := range options { - if !foundItem(validIndexOptions, kk) { - return fmt.Errorf("Unknown key: %v:%v", kk, vv) - } - } - default: - return fmt.Errorf("Unknown key: %v:%v", k, v) - } - } - return nil -} - -func foundItem(items []string, item string) bool { - for _, i := range items { - if item == i { - return true - } - } - return false -} - -type postIndexResponse struct{} - -// handleDeleteIndex handles DELETE /index request. -func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - err := h.API.DeleteIndex(r.Context(), indexName) - if err != nil { - h.Logger.Printf("problem deleting index: %s", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type deleteIndexResponse struct{} - -// handlePostIndex handles POST /index request. -func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - - // Decode request. - var req postIndexRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err == io.EOF { - // If no data was provided (EOF), we still create the index - // with default values. - } else if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - _, err = h.API.CreateIndex(r.Context(), indexName, req.Options) - if errors.Cause(err) == ErrIndexExists { - http.Error(w, err.Error(), http.StatusConflict) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -// handlePostIndexAttrDiff handles POST /index/attr/diff requests. -func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - - // Decode request. - var req postIndexAttrDiffRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks) - if err != nil { - if errors.Cause(err) == ErrIndexNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ - Attrs: attrs, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type postIndexAttrDiffRequest struct { - Blocks []AttrBlock `json:"blocks"` -} - -type postIndexAttrDiffResponse struct { - Attrs map[uint64]map[string]interface{} `json:"attrs"` -} - -// handlePostField handles POST /field request. -func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - fieldName := mux.Vars(r)["field"] - - // Decode request. - var req postFieldRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err == io.EOF { - // If no data was provided (EOF), we still create the field - // with default values. - } else if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - _, err = h.API.CreateField(r.Context(), indexName, fieldName, req.Options) - if err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrFieldExists: - http.Error(w, err.Error(), http.StatusConflict) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - // Encode response. - if err := json.NewEncoder(w).Encode(postFieldResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type _postFieldRequest postFieldRequest - -// Custom Unmarshal JSON to validate request body when creating a new field. If there's new FieldOptions, -// adding it to validFieldOptions to make sure the new option is validated, otherwise the request will be failed -func (p *postFieldRequest) UnmarshalJSON(b []byte) error { - // m is an overflow map used to capture additional, unexpected keys. - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return errors.Wrap(err, "unmarshaling unexpected keys") - } - - validFieldOptions := getValidOptions(FieldOptions{}) - err := validateOptions(m, validFieldOptions) - if err != nil { - return err - } - - // Unmarshal expected values. - var _p _postFieldRequest - if err := json.Unmarshal(b, &_p); err != nil { - return errors.Wrap(err, "unmarshalling expected keys") - } - - p.Options = _p.Options - return nil - -} - -func getValidOptions(option interface{}) []string { - validOptions := []string{} - val := reflect.ValueOf(option) - for i := 0; i < val.Type().NumField(); i++ { - jsonTag := val.Type().Field(i).Tag.Get("json") - s := strings.Split(jsonTag, ",") - validOptions = append(validOptions, s[0]) - } - return validOptions -} - -type postFieldRequest struct { - Options FieldOptions `json:"options"` -} - -type postFieldResponse struct{} - -// handleDeleteField handles DELETE /field request. -func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - fieldName := mux.Vars(r)["field"] - - err := h.API.DeleteField(r.Context(), indexName, fieldName) - if err != nil { - if errors.Cause(err) == ErrIndexNotFound { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } - return - } - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteFieldResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type deleteFieldResponse struct{} - -// handlePostFieldAttrDiff handles POST /field/attr/diff requests. -func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - fieldName := mux.Vars(r)["field"] - - // Decode request. - var req postFieldAttrDiffRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - attrs, err := h.API.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks) - if err != nil { - switch errors.Cause(err) { - case ErrFragmentNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{ - Attrs: attrs, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type postFieldAttrDiffRequest struct { - Blocks []AttrBlock `json:"blocks"` -} - -type postFieldAttrDiffResponse struct { - Attrs map[uint64]map[string]interface{} `json:"attrs"` -} - -// readQueryRequest parses an query parameters from r. -func (h *Handler) readQueryRequest(r *http.Request) (*QueryRequest, error) { - switch r.Header.Get("Content-Type") { - case "application/x-protobuf": - return h.readProtobufQueryRequest(r) - default: - return h.readURLQueryRequest(r) - } -} - -// readProtobufQueryRequest parses query parameters in protobuf from r. -func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, error) { - // Slurp the body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } - - // Unmarshal into object. - var req internal.QueryRequest - if err := proto.Unmarshal(body, &req); err != nil { - return nil, errors.Wrap(err, "unmarshalling") - } - - return decodeQueryRequest(&req), nil -} - -// readURLQueryRequest parses query parameters from URL parameters from r. -func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { - q := r.URL.Query() - - // Parse query string. - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } - query := string(buf) - - // Parse list of slices. - slices, err := parseUint64Slice(q.Get("slices")) - if err != nil { - return nil, errors.New("invalid slice argument") - } - - return &QueryRequest{ - Query: query, - Slices: slices, - ColumnAttrs: q.Get("columnAttrs") == "true", - ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", - ExcludeColumns: q.Get("excludeColumns") == "true", - }, nil -} - -// writeQueryResponse writes the response from the executor to w. -func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *QueryResponse) error { - if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { - return h.writeProtobufQueryResponse(w, resp) - } - return h.writeJSONQueryResponse(w, resp) -} - -// writeProtobufQueryResponse writes the response from the executor to w as protobuf. -func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *QueryResponse) error { - if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil { - return errors.Wrap(err, "marshalling") - } else if _, err := w.Write(buf); err != nil { - return errors.Wrap(err, "writing") - } - return nil -} - -// writeJSONQueryResponse writes the response from the executor to w as JSON. -func (h *Handler) writeJSONQueryResponse(w http.ResponseWriter, resp *QueryResponse) error { - return json.NewEncoder(w).Encode(resp) -} - -// handlePostImport handles /import requests. -func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Marshal into request object. - var req internal.ImportRequest - if err := proto.Unmarshal(body, &req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.API.Import(r.Context(), req); err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - fallthrough - case ErrFieldNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) - if e != nil { - http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) - return - } - - // Write response. - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - } - w.Write(buf) -} - -// handlePostImportValue handles /import-value requests. -func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Marshal into request object. - var req internal.ImportValueRequest - if err := proto.Unmarshal(body, &req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err = h.API.ImportValue(r.Context(), req); err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - fallthrough - case ErrFieldNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) - if e != nil { - http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) - return - } - - // Write response. - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - } - w.Write(buf) -} - -// handleGetExport handles /export requests. -func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { - switch r.Header.Get("Accept") { - case "text/csv": - h.handleGetExportCSV(w, r) - default: - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - } -} - -func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { - // Parse query parameters. - q := r.URL.Query() - index, field := q.Get("index"), q.Get("field") - - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "invalid slice", http.StatusBadRequest) - return - } - - if err = h.API.ExportCSV(r.Context(), index, field, slice, w); err != nil { - switch errors.Cause(err) { - case ErrFragmentNotFound: - break - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } -} - -// handleGetFragmentNodes handles /fragment/nodes requests. -func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { - q := r.URL.Query() - index := q.Get("index") - - // Read slice parameter. - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest) - return - } - - // Retrieve fragment owner nodes. - nodes, err := h.API.SliceNodes(r.Context(), index, slice) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Write to response. - if err := json.NewEncoder(w).Encode(nodes); err != nil { - h.Logger.Printf("json write error: %s", err) - } -} - -// handleGetFragmentBlockData handles GET /fragment/block/data requests. -func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { - buf, err := h.API.FragmentBlockData(r.Context(), r.Body) - if err != nil { - if _, ok := err.(BadRequestError); ok { - http.Error(w, err.Error(), http.StatusBadRequest) - } else if errors.Cause(err) == ErrFragmentNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Write response. - w.Header().Set("Content-Type", "application/protobuf") - w.Header().Set("Content-Length", strconv.Itoa(len(buf))) - w.Write(buf) -} - -// handleGetFragmentBlocks handles GET /fragment/blocks requests. -func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) { - // Read slice parameter. - q := r.URL.Query() - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) - return - } - - blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), slice) - if err != nil { - if errors.Cause(err) == ErrFragmentNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ - Blocks: blocks, - }); err != nil { - h.Logger.Printf("block response encoding error: %s", err) - } -} - -type getFragmentBlocksResponse struct { - Blocks []FragmentBlock `json:"blocks"` -} - -// handleGetVersion handles /version requests. -func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { - err := json.NewEncoder(w).Encode(struct { - Version string `json:"version"` - }{ - Version: h.API.Version(), - }) - if err != nil { - h.Logger.Printf("write version response error: %s", err) - } -} - -// QueryResult types. -const ( - QueryResultTypeNil uint32 = iota - QueryResultTypeRow - QueryResultTypePairs - QueryResultTypeValCount - QueryResultTypeUint64 - QueryResultTypeBool ) // QueryRequest represent a request to process a query. @@ -970,19 +31,6 @@ type QueryRequest struct { Remote bool } -func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { - req := &QueryRequest{ - Query: pb.Query, - Slices: pb.Slices, - ColumnAttrs: pb.ColumnAttrs, - Remote: pb.Remote, - ExcludeRowAttrs: pb.ExcludeRowAttrs, - ExcludeColumns: pb.ExcludeColumns, - } - - return req -} - // QueryResponse represent a response from a processed query. type QueryResponse struct { // Result for each top-level query call. @@ -1012,234 +60,19 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { return json.Marshal(output) } -func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { - pb := &internal.QueryResponse{ - Results: make([]*internal.QueryResult, len(resp.Results)), - ColumnAttrSets: encodeColumnAttrSets(resp.ColumnAttrSets), - } - - for i := range resp.Results { - pb.Results[i] = &internal.QueryResult{} - - switch result := resp.Results[i].(type) { - case *Row: - pb.Results[i].Type = QueryResultTypeRow - pb.Results[i].Row = encodeRow(result) - case []Pair: - pb.Results[i].Type = QueryResultTypePairs - pb.Results[i].Pairs = encodePairs(result) - case ValCount: - pb.Results[i].Type = QueryResultTypeValCount - pb.Results[i].ValCount = encodeValCount(result) - case uint64: - pb.Results[i].Type = QueryResultTypeUint64 - pb.Results[i].N = result - case bool: - pb.Results[i].Type = QueryResultTypeBool - pb.Results[i].Changed = result - case nil: - pb.Results[i].Type = QueryResultTypeNil - } - } - - if resp.Err != nil { - pb.Err = resp.Err.Error() - } - - return pb +type Handlerer interface { + http.Handler + GetAPI() *API } -// parseUint64Slice returns a slice of uint64s from a comma-delimited string. -func parseUint64Slice(s string) ([]uint64, error) { - var a []uint64 - for _, str := range strings.Split(s, ",") { - // Ignore blanks. - if str == "" { - continue - } +type NopHandler struct{} - // Parse number. - num, err := strconv.ParseUint(str, 10, 64) - if err != nil { - return nil, errors.Wrap(err, "parsing int") - } - a = append(a, num) - } - return a, nil -} +func (n *NopHandler) ServeHTTP(_ http.ResponseWriter, _ *http.Request) {} -// errorString returns the string representation of err. -func errorString(err error) string { - if err == nil { - return "" - } - return err.Error() -} - -func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { - // Decode request. - var req setCoordinatorRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest) - return - } - - oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID) - if err != nil { - if errors.Cause(err) == ErrNodeIDNotExists { - http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) - } else { - http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError) - } - return - } - // Encode response. - if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ - Old: oldNode, - New: newNode, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type setCoordinatorRequest struct { - ID string `json:"id"` -} - -type setCoordinatorResponse struct { - Old *Node `json:"old"` - New *Node `json:"new"` -} - -// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. -func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) { - // Decode request. - var req removeNodeRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - removeNode, err := h.API.RemoveNode(req.ID) - if err != nil { - if errors.Cause(err) == ErrNodeIDNotExists { - http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) - } else { - http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(removeNodeResponse{ - Remove: removeNode, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type removeNodeRequest struct { - ID string `json:"id"` -} - -type removeNodeResponse struct { - Remove *Node `json:"remove"` -} - -// handlePostClusterResizeAbort handles POST /cluster/resize/abort request. -func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { - err := h.API.ResizeAbort() - var msg string - if err != nil { - switch errors.Cause(err) { - case ErrNodeNotCoordinator: - http.Error(w, err.Error(), http.StatusBadRequest) - return - case ErrResizeNotRunning: - msg = err.Error() - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - // Encode response. - if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ - Info: msg, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type clusterResizeAbortResponse struct { - Info string `json:"info"` -} - -func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { - err := h.API.RecalculateCaches(r.Context()) - if err != nil { - http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) -} - -func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } - - err := h.API.ClusterMessage(r.Context(), r.Body) - if err != nil { - // TODO this was the previous behavior, but perhaps not everything is a bad request - http.Error(w, err.Error(), http.StatusBadRequest) - } - - if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type defaultClusterMessageResponse struct{} - -type queryValidationSpec struct { - required []string - args map[string]struct{} -} - -func queryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { - args := map[string]struct{}{} - for _, arg := range requiredArgs { - args[arg] = struct{}{} - } - - return &queryValidationSpec{ - required: requiredArgs, - args: args, - } -} - -func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { - for _, arg := range args { - s.args[arg] = struct{}{} - } - return s -} - -func (s queryValidationSpec) validate(query url.Values) error { - for _, req := range s.required { - if query.Get(req) == "" { - return errors.Errorf("%s is required", req) - } - } - for k := range query { - if _, ok := s.args[k]; !ok { - return errors.Errorf("%s is not a valid argument", k) - } - } +func (n *NopHandler) GetAPI() *API { return nil } + +func NewNopHandler() Handlerer { + return &NopHandler{} +} diff --git a/holder.go b/holder.go index e8db140ce..2330d59e7 100644 --- a/holder.go +++ b/holder.go @@ -662,11 +662,9 @@ func (s *HolderSyncer) syncIndex(index string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.ColumnAttrDiff(context.Background(), index, blks) + m, err := s.Cluster.InternalClient.ColumnAttrDiff(context.Background(), &node.URI, index, blks) if err != nil { return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { @@ -708,11 +706,9 @@ func (s *HolderSyncer) syncField(index, name string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.RowAttrDiff(context.Background(), index, name, blks) + m, err := s.Cluster.InternalClient.RowAttrDiff(context.Background(), &node.URI, index, name, blks) if err == ErrFieldNotFound { continue // field not created remotely yet, skip } else if err != nil { diff --git a/holder_test.go b/holder_test.go index 5f41cfab7..70fecf245 100644 --- a/holder_test.go +++ b/holder_test.go @@ -24,8 +24,8 @@ import ( "testing" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -360,8 +360,20 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { + s := test.NewServer() + defer s.Close() + + uri, err := pilosa.NewURIFromAddress(s.URL) + if err != nil { + t.Fatal(err) + } + cluster := test.NewCluster(2) - client := server.GetHTTPClient(nil) + client := http.GetHTTPClient(nil) + httpClient := http.NewInternalHTTPClientFromURI(uri, client) + cluster.InternalClient = httpClient + cluster.RemoteClient = client + // Create a local holder. hldr0 := test.MustOpenHolder() defer hldr0.Close() @@ -369,11 +381,9 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Create a remote holder wrapped by an HTTP hldr1 := test.MustOpenHolder() defer hldr1.Close() - s := test.NewServer() - defer s.Close() s.Handler.API.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(client) + e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) e.Holder = hldr1.Holder e.Node = cluster.Nodes[1] e.Cluster = cluster @@ -383,11 +393,6 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Mock 2-node, fully replicated cluster. cluster.ReplicaN = 2 - uri, err := pilosa.NewURIFromAddress(s.URL) - if err != nil { - t.Fatal(err) - } - cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) cluster.Nodes[1].URI = *uri @@ -445,7 +450,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { Holder: hldr0.Holder, Node: cluster.Nodes[0], Cluster: cluster, - RemoteClient: server.GetHTTPClient(nil), + RemoteClient: http.GetHTTPClient(nil), Stats: pilosa.NopStatsClient, } diff --git a/http/client.go b/http/client.go new file mode 100644 index 000000000..b0fee7887 --- /dev/null +++ b/http/client.go @@ -0,0 +1,1034 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "net/url" + "sort" + "strconv" + + "crypto/tls" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" +) + +// ClientOptions represents the configuration for a InternalHTTPClient +type ClientOptions struct { + TLS *tls.Config +} + +// InternalHTTPClient represents a client to the Pilosa cluster. +type InternalHTTPClient struct { + defaultURI *pilosa.URI + + // The client to use for HTTP communication. + HTTPClient *http.Client +} + +// NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host. +func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTTPClient, error) { + if host == "" { + return nil, pilosa.ErrHostRequired + } + + uri, err := pilosa.NewURIFromAddress(host) + if err != nil { + return nil, errors.Wrap(err, "getting URI") + } + + client := NewInternalHTTPClientFromURI(uri, remoteClient) + return client, nil +} + +func NewInternalHTTPClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalHTTPClient { + return &InternalHTTPClient{ + defaultURI: defaultURI, + HTTPClient: remoteClient, + } +} + +// Host returns the host the client was initialized with. +func (c *InternalHTTPClient) Host() *pilosa.URI { return c.defaultURI } + +// MaxSliceByIndex returns the number of slices on a server by index. +func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { + return c.maxSliceByIndex(ctx) +} + +// maxSliceByIndex returns the number of slices on a server by index. +func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { + // Execute request against the host. + u := uriPathToURL(c.defaultURI, "/slices/max") + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var rsp getSlicesMaxResponse + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + + return rsp.Standard, nil +} + +// Schema returns all index and field schema information. +func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) { + // Execute request against the host. + u := c.defaultURI.Path("/schema") + + // Build request. + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var rsp getSchemaResponse + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + return rsp.Indexes, nil +} + +// CreateIndex creates a new index on the server. +func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error { + // Encode query request. + buf, err := json.Marshal(&postIndexRequest{ + Options: opt, + }) + if err != nil { + return errors.Wrap(err, "encoding request") + } + + // Create URL & HTTP request. + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index)) + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + return nil // ok + case http.StatusConflict: + return pilosa.ErrIndexExists + default: + return errors.New(string(body)) + } +} + +// FragmentNodes returns a list of nodes that own a slice. +func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*pilosa.Node, error) { + // Execute request against the host. + u := uriPathToURL(c.defaultURI, "/fragment/nodes") + u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var a []*pilosa.Node + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&a); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + + return a, nil +} + +// Query executes query against the index. +func (c *InternalHTTPClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return c.QueryNode(ctx, c.defaultURI, index, queryRequest) +} + +// QueryNode executes query against the index, sending the request to the node specified. +func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + if index == "" { + return nil, pilosa.ErrIndexRequired + } else if queryRequest.Query == "" { + return nil, pilosa.ErrQueryRequired + } + + // Encode request object. + buf, err := proto.Marshal(queryRequest) + if err != nil { + return nil, errors.Wrap(err, "marshaling") + } + + // Create HTTP request. + u := uri.Path(fmt.Sprintf("/index/%s/query", index)) + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } else if resp.StatusCode != http.StatusOK { + return nil, errors.New(string(body)) + } + + qresp := &internal.QueryResponse{} + if err := proto.Unmarshal(body, qresp); err != nil { + return nil, fmt.Errorf("unmarshal response: %s", err) + } else if s := qresp.Err; s != "" { + return nil, errors.New(s) + } + + return qresp, nil +} + +// Import bulk imports bits for a single slice to a host. +func (c *InternalHTTPClient) Import(ctx context.Context, index, field string, slice uint64, bits []pilosa.Bit) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := marshalImportPayload(index, field, slice, bits) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + // Retrieve a list of nodes that own the slice. + nodes, err := c.FragmentNodes(ctx, index, slice) + if err != nil { + return fmt.Errorf("slice nodes: %s", err) + } + + // Import to each node. + for _, node := range nodes { + if err := c.importNode(ctx, node, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + } + + return nil +} + +// ImportK bulk imports bits specified by string keys to a host. +func (c *InternalHTTPClient) ImportK(ctx context.Context, index, field string, columns []pilosa.Bit) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := marshalImportPayloadK(index, field, columns) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + node := &pilosa.Node{ + URI: *c.defaultURI, + } + + // Import to node. + if err := c.importNode(ctx, node, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + + return nil +} + +func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, options pilosa.IndexOptions) error { + err := c.CreateIndex(ctx, name, options) + if err == nil || err == pilosa.ErrIndexExists { + return nil + } + return err +} + +func (c *InternalHTTPClient) EnsureField(ctx context.Context, indexName string, fieldName string, options pilosa.FieldOptions) error { + err := c.CreateField(ctx, indexName, fieldName, options) + if err == nil || err == pilosa.ErrFieldExists { + return nil + } + return err +} + +// marshalImportPayload marshalls the import parameters into a protobuf byte slice. +func marshalImportPayload(index, field string, slice uint64, bits []pilosa.Bit) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + rowIDs := Bits(bits).RowIDs() + columnIDs := Bits(bits).ColumnIDs() + timestamps := Bits(bits).Timestamps() + + // Marshal data to protobuf. + buf, err := proto.Marshal(&internal.ImportRequest{ + Index: index, + Field: field, + Slice: slice, + RowIDs: rowIDs, + ColumnIDs: columnIDs, + Timestamps: timestamps, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. +func marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + rowKeys := Bits(bits).RowKeys() + columnKeys := Bits(bits).ColumnKeys() + timestamps := Bits(bits).Timestamps() + + // Marshal data to protobuf. + buf, err := proto.Marshal(&internal.ImportRequest{ + Index: index, + Field: field, + RowKeys: rowKeys, + ColumnKeys: columnKeys, + Timestamps: timestamps, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// importNode sends a pre-marshaled import request to a node. +func (c *InternalHTTPClient) importNode(ctx context.Context, node *pilosa.Node, buf []byte) error { + // Create URL & HTTP request. + u := nodePathToURL(node, "/import") + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } else if resp.StatusCode != http.StatusOK { + return errors.New(string(body)) + } + + var isresp internal.ImportResponse + if err := proto.Unmarshal(body, &isresp); err != nil { + return fmt.Errorf("unmarshal import response: %s", err) + } else if s := isresp.Err; s != "" { + return errors.New(s) + } + + return nil +} + +// ImportValue bulk imports field values for a single slice to a host. +func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []pilosa.FieldValue) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := marshalImportValuePayload(index, field, slice, vals) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + // Retrieve a list of nodes that own the slice. + nodes, err := c.FragmentNodes(ctx, index, slice) + if err != nil { + return fmt.Errorf("slice nodes: %s", err) + } + + // Import to each node. + for _, node := range nodes { + if err := c.importValueNode(ctx, node, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + } + + return nil +} + +// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. +func marshalImportValuePayload(index, field string, slice uint64, vals []pilosa.FieldValue) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + columnIDs := FieldValues(vals).ColumnIDs() + values := FieldValues(vals).Values() + + // Marshal data to protobuf. + buf, err := proto.Marshal(&internal.ImportValueRequest{ + Index: index, + Field: field, + Slice: slice, + ColumnIDs: columnIDs, + Values: values, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// importValueNode sends a pre-marshaled import request to a node. +func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *pilosa.Node, buf []byte) error { + // Create URL & HTTP request. + u := nodePathToURL(node, "/import-value") + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } else if resp.StatusCode != http.StatusOK { + return errors.New(string(body)) + } + + var isresp internal.ImportResponse + if err := proto.Unmarshal(body, &isresp); err != nil { + return fmt.Errorf("unmarshal import response: %s", err) + } else if s := isresp.Err; s != "" { + return errors.New(s) + } + + return nil +} + +// ExportCSV bulk exports data for a single slice from a host to CSV format. +func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + // Retrieve a list of nodes that own the slice. + nodes, err := c.FragmentNodes(ctx, index, slice) + if err != nil { + return fmt.Errorf("slice nodes: %s", err) + } + + // Attempt nodes in random order. + var e error + for _, i := range rand.Perm(len(nodes)) { + node := nodes[i] + + if err := c.exportNodeCSV(ctx, node, index, field, slice, w); err != nil { + e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) + continue + } else { + return nil + } + } + + return e +} + +// exportNode copies a CSV export from a node to w. +func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, slice uint64, w io.Writer) error { + // Create URL. + u := nodePathToURL(node, "/export") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "slice": {strconv.FormatUint(slice, 10)}, + }.Encode() + + // Generate HTTP request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Accept", "text/csv") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Validate status code. + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("invalid status: %d", resp.StatusCode) + } + + // Copy body to writer. + if _, err := io.Copy(w, resp.Body); err != nil { + return errors.Wrap(err, "copying") + } + + return nil +} + +func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri pilosa.URI) (io.ReadCloser, error) { + node := &pilosa.Node{ + URI: uri, + } + return c.backupSliceNode(ctx, index, field, slice, node) +} + +func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *pilosa.Node) (io.ReadCloser, error) { + u := nodePathToURL(node, "/fragment/data") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "slice": {strconv.FormatUint(slice, 10)}, + }.Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + + // Return error if status is not OK. + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + return nil, pilosa.ErrFragmentNotFound + } else if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode) + } + + return resp.Body, nil +} + +// CreateField creates a new field on the server. +func (c *InternalHTTPClient) CreateField(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { + if index == "" { + return pilosa.ErrIndexRequired + } + + // Encode query request. + buf, err := json.Marshal(&postFieldRequest{ + Options: opt, + }) + if err != nil { + return errors.Wrap(err, "marshaling") + } + + // Create URL & HTTP request. + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s", index, field)) + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + return nil // ok + case http.StatusConflict: + return pilosa.ErrFieldExists + default: + return errors.New(string(body)) + } +} + +// FragmentBlocks returns a list of block checksums for a fragment on a host. +// Only returns blocks which contain data. +func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64) ([]pilosa.FragmentBlock, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, "/fragment/blocks") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "slice": {strconv.FormatUint(slice, 10)}, + }.Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + case http.StatusNotFound: + return nil, pilosa.ErrFragmentNotFound + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp getFragmentBlocksResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Blocks, nil +} + +// BlockData returns row/column id pairs for a block. +func (c *InternalHTTPClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { + buf, err := proto.Marshal(&internal.BlockDataRequest{ + Index: index, + Field: field, + Slice: slice, + Block: uint64(block), + }) + if err != nil { + return nil, nil, errors.Wrap(err, "marshaling") + } + + u := uriPathToURL(c.defaultURI, "/fragment/block/data") + req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/protobuf") + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Accept", "application/protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // fallthrough + case http.StatusNotFound: + return nil, nil, nil + default: + return nil, nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp internal.BlockDataResponse + if body, err := ioutil.ReadAll(resp.Body); err != nil { + return nil, nil, errors.Wrap(err, "reading") + } else if err := proto.Unmarshal(body, &rsp); err != nil { + return nil, nil, errors.Wrap(err, "unmarshalling") + } + return rsp.RowIDs, rsp.ColumnIDs, nil +} + +// ColumnAttrDiff returns data from differing blocks on a remote host. +func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, fmt.Sprintf("/index/%s/attr/diff", index)) + + // Encode request. + buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) + if err != nil { + return nil, errors.Wrap(err, "marshaling") + } + + // Build request. + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp postIndexAttrDiffResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Attrs, nil +} + +// RowAttrDiff returns data from differing blocks on a remote host. +func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, fmt.Sprintf("/index/%s/field/%s/attr/diff", index, field)) + + // Encode request. + buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks}) + if err != nil { + return nil, errors.Wrap(err, "marshaling") + } + + // Build request. + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + case http.StatusNotFound: + return nil, pilosa.ErrFieldNotFound + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp postFieldAttrDiffResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Attrs, nil +} + +// SendMessage posts a message synchronously. +func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb proto.Message) error { + msg, err := pilosa.MarshalMessage(pb) + if err != nil { + return fmt.Errorf("marshaling message: %v", err) + } + + u := uriPathToURL(uri, "/cluster/message") + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) + if err != nil { + return errors.Wrap(err, "making new request") + } + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return fmt.Errorf("executing http request: %v", err) + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response body: %v", err) + } + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + default: + return fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body) + } + + return nil +} + +// Bits is a slice of Bit. +type Bits []pilosa.Bit + +func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Bits) Len() int { return len(p) } + +func (p Bits) Less(i, j int) bool { + if p[i].RowID == p[j].RowID { + if p[i].ColumnID < p[j].ColumnID { + return p[i].Timestamp < p[j].Timestamp + } + return p[i].ColumnID < p[j].ColumnID + } + return p[i].RowID < p[j].RowID +} + +// RowIDs returns a slice of all the row IDs. +func (p Bits) RowIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].RowID + } + return other +} + +// ColumnIDs returns a slice of all the column IDs. +func (p Bits) ColumnIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].ColumnID + } + return other +} + +// RowKeys returns a slice of all the row keys. +func (p Bits) RowKeys() []string { + other := make([]string, len(p)) + for i := range p { + other[i] = p[i].RowKey + } + return other +} + +// ColumnKeys returns a slice of all the column keys. +func (p Bits) ColumnKeys() []string { + other := make([]string, len(p)) + for i := range p { + other[i] = p[i].ColumnKey + } + return other +} + +// Timestamps returns a slice of all the timestamps. +func (p Bits) Timestamps() []int64 { + other := make([]int64, len(p)) + for i := range p { + other[i] = p[i].Timestamp + } + return other +} + +// GroupBySlice returns a map of bits by slice. +func (p Bits) GroupBySlice() map[uint64][]pilosa.Bit { + m := make(map[uint64][]pilosa.Bit) + for _, bit := range p { + slice := bit.ColumnID / pilosa.SliceWidth + m[slice] = append(m[slice], bit) + } + + for slice, bits := range m { + sort.Sort(Bits(bits)) + m[slice] = bits + } + + return m +} + +// FieldValues represents a slice of field values. +type FieldValues []pilosa.FieldValue + +func (p FieldValues) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p FieldValues) Len() int { return len(p) } + +func (p FieldValues) Less(i, j int) bool { + return p[i].ColumnID < p[j].ColumnID +} + +// ColumnIDs returns a slice of all the column IDs. +func (p FieldValues) ColumnIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].ColumnID + } + return other +} + +// Values returns a slice of all the values. +func (p FieldValues) Values() []int64 { + other := make([]int64, len(p)) + for i := range p { + other[i] = p[i].Value + } + return other +} + +// GroupBySlice returns a map of field values by slice. +func (p FieldValues) GroupBySlice() map[uint64][]pilosa.FieldValue { + m := make(map[uint64][]pilosa.FieldValue) + for _, val := range p { + slice := val.ColumnID / pilosa.SliceWidth + m[slice] = append(m[slice], val) + } + + for slice, vals := range m { + sort.Sort(FieldValues(vals)) + m[slice] = vals + } + + return m +} + +// BitsByPos is a slice of bits sorted row then column. +type BitsByPos []pilosa.Bit + +func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p BitsByPos) Len() int { return len(p) } +func (p BitsByPos) Less(i, j int) bool { + p0, p1 := pilosa.Pos(p[i].RowID, p[i].ColumnID), pilosa.Pos(p[j].RowID, p[j].ColumnID) + if p0 == p1 { + return p[i].Timestamp < p[j].Timestamp + } + return p0 < p1 +} + +func uriPathToURL(uri *pilosa.URI, path string) url.URL { + return url.URL{ + Scheme: uri.Scheme(), + Host: uri.HostPort(), + Path: path, + } +} + +func nodePathToURL(node *pilosa.Node, path string) url.URL { + return url.URL{ + Scheme: node.URI.Scheme(), + Host: node.URI.HostPort(), + Path: path, + } +} diff --git a/client_test.go b/http/client_test.go similarity index 93% rename from client_test.go rename to http/client_test.go index d2e194805..93b16a177 100644 --- a/client_test.go +++ b/http/client_test.go @@ -12,20 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package http_test import ( "context" "fmt" - "net/http" + gohttp "net/http" "reflect" "testing" "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -43,10 +43,10 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { return server, hldr } -var defaultClient *http.Client +var defaultClient *gohttp.Client func init() { - defaultClient = server.GetHTTPClient(nil) + defaultClient = http.GetHTTPClient(nil) } @@ -61,21 +61,24 @@ func TestClient_MultiNode(t *testing.T) { } s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) + httpClient := http.NewInternalHTTPClientFromURI(&cluster.Nodes[0].URI, defaultClient) + e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) e.Holder = hldr[0].Holder e.Node = cluster.Nodes[0] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) + httpClient := http.NewInternalHTTPClientFromURI(&cluster.Nodes[0].URI, defaultClient) + e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) e.Holder = hldr[1].Holder e.Node = cluster.Nodes[1] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) + httpClient := http.NewInternalHTTPClientFromURI(&cluster.Nodes[0].URI, defaultClient) + e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) e.Holder = hldr[2].Holder e.Node = cluster.Nodes[2] e.Cluster = cluster @@ -98,9 +101,9 @@ func TestClient_MultiNode(t *testing.T) { } } - baseBit0 := SliceWidth * sliceNums[0] - baseBit1 := SliceWidth * sliceNums[1] - baseBit2 := SliceWidth * sliceNums[2] + baseBit0 := pilosa.SliceWidth * sliceNums[0] + baseBit1 := pilosa.SliceWidth * sliceNums[1] + baseBit2 := pilosa.SliceWidth * sliceNums[2] maxSlice := uint64(0) for _, x := range sliceNums { @@ -336,7 +339,7 @@ func TestClient_FragmentBlocks(t *testing.T) { // Retrieve blocks. c := test.MustNewClient(s.Host(), defaultClient) - blocks, err := c.FragmentBlocks(context.Background(), "i", "f", 0) + blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0) if err != nil { t.Fatal(err) } else if len(blocks) != 2 { diff --git a/http/handler.go b/http/handler.go new file mode 100644 index 000000000..df5108826 --- /dev/null +++ b/http/handler.go @@ -0,0 +1,1231 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "crypto/tls" + "encoding/json" + "expvar" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + // Imported for its side-effect of registering pprof endpoints with the server. + _ "net/http/pprof" + "reflect" + "runtime/debug" + "strconv" + "strings" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/gorilla/handlers" + "github.com/gorilla/mux" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + + "github.com/pkg/errors" +) + +// Handler represents an HTTP handler. +type Handler struct { + Handler http.Handler + + Logger pilosa.Logger + + // Keeps the query argument validators for each handler + validators map[string]*queryValidationSpec + + API *pilosa.API + + AllowedOrigins []string +} + +// externalPrefixFlag denotes endpoints that are intended to be exposed to clients. +// This is used for stats tagging. +var externalPrefixFlag = map[string]bool{ + "schema": true, + "query": true, + "import": true, + "export": true, + "index": true, + "field": true, + "nodes": true, + "version": true, +} + +type errorResponse struct { + Error string `json:"error"` +} + +// HandlerOption is a functional option type for pilosa.Handler +type HandlerOption func(s *Handler) error + +func OptHandlerAllowedOrigins(origins []string) HandlerOption { + return func(h *Handler) error { + h.Handler = handlers.CORS( + handlers.AllowedOrigins(origins), + handlers.AllowedHeaders([]string{"Content-Type"}), + )(h.Handler) + return nil + } +} + +func OptHandlerAPI(api *pilosa.API) HandlerOption { + return func(h *Handler) error { + h.API = api + return nil + } +} + +func OptHandlerLogger(logger pilosa.Logger) HandlerOption { + return func(h *Handler) error { + h.Logger = logger + return nil + } +} + +// NewHandler returns a new instance of Handler with a default logger. +func NewHandler(opts ...HandlerOption) (*Handler, error) { + handler := &Handler{ + Logger: pilosa.NopLogger, + } + handler.Handler = NewRouter(handler) + handler.populateValidators() + + for _, opt := range opts { + err := opt(handler) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + + return handler, nil +} + +func (h *Handler) populateValidators() { + h.validators = map[string]*queryValidationSpec{} + h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") + h.validators["GetSliceMax"] = queryValidationSpecRequired() + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") + h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "slice") +} + +func (h *Handler) queryArgValidator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := mux.CurrentRoute(r).GetName() + if validator, ok := h.validators[key]; ok { + if err := validator.validate(r.URL.Query()); err != nil { + // TODO: Return the response depending on the Accept header + response := errorResponse{Error: err.Error()} + body, err := json.Marshal(response) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + http.Error(w, string(body), http.StatusBadRequest) + return + } + } + next.ServeHTTP(w, r) + }) +} + +// NewRouter creates a new mux http router. +func NewRouter(handler *Handler) *mux.Router { + router := mux.NewRouter() + router.HandleFunc("/", handler.handleHome).Methods("GET") + router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") + router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") + router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") + router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client + router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") + router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") + router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") + + router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") + + router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") + router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") + router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") + router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") + router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") + router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") + router.HandleFunc("/import", handler.handlePostImport).Methods("POST") + router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") + router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") + router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") + router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") + router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") + router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") + //router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented. + router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") + router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") + router.HandleFunc("/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST") + router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") + router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") + + // TODO: Apply MethodNotAllowed statuses to all endpoints. + // Ideally this would be automatic, as described in this (wontfix) ticket: + // https://github.com/gorilla/mux/issues/6 + // For now we just do it for the most commonly used handler, /query + router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") + + router.Use(handler.queryArgValidator) + return router +} + +func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) +} + +// ServeHTTP handles an HTTP request. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + stack := debug.Stack() + msg := "PANIC: %s\n%s" + h.Logger.Printf(msg, err, stack) + fmt.Fprintf(w, msg, err, stack) + } + }() + + t := time.Now() + h.Handler.ServeHTTP(w, r) + dif := time.Since(t) + + // Calculate per request StatsD metrics when the handler is fully configured. + statsTags := make([]string, 0, 3) + + longQueryTime := h.API.LongQueryTime() + if longQueryTime > 0 && dif > longQueryTime { + h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) + statsTags = append(statsTags, "slow_query") + } + + pathParts := strings.Split(r.URL.Path, "/") + endpointName := strings.Join(pathParts, "_") + + if externalPrefixFlag[pathParts[1]] { + statsTags = append(statsTags, "external") + } + + // useragent tag identifies internal/external endpoints + statsTags = append(statsTags, "useragent:"+r.UserAgent()) + stats := h.API.StatsWithTags(statsTags) + if stats != nil { + stats.Histogram("http."+endpointName, float64(dif), 0.1) + } +} + +func (h *Handler) handleHome(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.", http.StatusNotFound) +} + +// handleGetSchema handles GET /schema requests. +func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { + schema := h.API.Schema(r.Context()) + if err := json.NewEncoder(w).Encode(getSchemaResponse{ + Indexes: schema, + }); err != nil { + h.Logger.Printf("write schema response error: %s", err) + } +} + +// handleGetStatus handles GET /status requests. +func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { + status := getStatusResponse{ + State: h.API.State(), + Nodes: h.API.Hosts(r.Context()), + LocalID: h.API.LocalID(), + } + if err := json.NewEncoder(w).Encode(status); err != nil { + h.Logger.Printf("write status response error: %s", err) + } +} + +func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { + info := h.API.Info() + if err := json.NewEncoder(w).Encode(info); err != nil { + h.Logger.Printf("write info response error: %s", err) + } +} + +type getSchemaResponse struct { + Indexes []*pilosa.IndexInfo `json:"indexes"` +} + +type getStatusResponse struct { + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + LocalID string `json:"localID"` +} + +// handlePostQuery handles /query requests. +func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { + // Parse incoming request. + req, err := h.readQueryRequest(r) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + return + } + // TODO: Remove + req.Index = mux.Vars(r)["index"] + + resp, err := h.API.Query(r.Context(), req) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + return + } + + // Set appropriate status code, if there is an error. + if resp.Err != nil { + switch resp.Err { + case pilosa.ErrTooManyWrites: + w.WriteHeader(http.StatusRequestEntityTooLarge) + default: + w.WriteHeader(http.StatusInternalServerError) + } + } + + // Write response back to client. + if err := h.writeQueryResponse(w, r, &resp); err != nil { + h.Logger.Printf("write query response error: %s", err) + } +} + +// handleGetSlicesMax handles GET /schema requests. +func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { + if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ + Standard: h.API.MaxSlices(r.Context()), + }); err != nil { + h.Logger.Printf("write slices-max response error: %s", err) + } +} + +type getSlicesMaxResponse struct { + Standard map[string]uint64 `json:"standard"` +} + +// handleGetIndexes handles GET /index request. +func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { + h.handleGetSchema(w, r) +} + +// handleGetIndex handles GET /index/ requests. +func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + index, err := h.API.Index(r.Context(), indexName) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + if err := json.NewEncoder(w).Encode(getIndexResponse{ + map[string]string{"name": index.Name()}, + }); err != nil { + h.Logger.Printf("write response error: %s", err) + } +} + +type getIndexResponse struct { + Index map[string]string `json:"index"` +} + +type postIndexRequest struct { + Options pilosa.IndexOptions `json:"options"` +} + +//_postIndexRequest is necessary to avoid recursion while decoding. +type _postIndexRequest postIndexRequest + +// Custom Unmarshal JSON to validate request body when creating a new index. +func (p *postIndexRequest) UnmarshalJSON(b []byte) error { + + // m is an overflow map used to capture additional, unexpected keys. + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return errors.Wrap(err, "unmarshalling unexpected values") + } + + validIndexOptions := getValidOptions(pilosa.IndexOptions{}) + err := validateOptions(m, validIndexOptions) + if err != nil { + return err + } + // Unmarshal expected values. + var _p _postIndexRequest + if err := json.Unmarshal(b, &_p); err != nil { + return errors.Wrap(err, "unmarshalling expected values") + } + + p.Options = _p.Options + + return nil +} + +// Raise errors for any unknown key +func validateOptions(data map[string]interface{}, validIndexOptions []string) error { + for k, v := range data { + switch k { + case "options": + options, ok := v.(map[string]interface{}) + if !ok { + return errors.New("options is not map[string]interface{}") + } + for kk, vv := range options { + if !foundItem(validIndexOptions, kk) { + return fmt.Errorf("Unknown key: %v:%v", kk, vv) + } + } + default: + return fmt.Errorf("Unknown key: %v:%v", k, v) + } + } + return nil +} + +func foundItem(items []string, item string) bool { + for _, i := range items { + if item == i { + return true + } + } + return false +} + +type postIndexResponse struct{} + +// handleDeleteIndex handles DELETE /index request. +func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + err := h.API.DeleteIndex(r.Context(), indexName) + if err != nil { + h.Logger.Printf("problem deleting index: %s", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type deleteIndexResponse struct{} + +// handlePostIndex handles POST /index request. +func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + + // Decode request. + var req postIndexRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err == io.EOF { + // If no data was provided (EOF), we still create the index + // with default values. + } else if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + _, err = h.API.CreateIndex(r.Context(), indexName, req.Options) + if errors.Cause(err) == pilosa.ErrIndexExists { + http.Error(w, err.Error(), http.StatusConflict) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +// handlePostIndexAttrDiff handles POST /index/attr/diff requests. +func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + + // Decode request. + var req postIndexAttrDiffRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ + Attrs: attrs, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type postIndexAttrDiffRequest struct { + Blocks []pilosa.AttrBlock `json:"blocks"` +} + +type postIndexAttrDiffResponse struct { + Attrs map[uint64]map[string]interface{} `json:"attrs"` +} + +// handlePostField handles POST /field request. +func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + // Decode request. + var req postFieldRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err == io.EOF { + // If no data was provided (EOF), we still create the field + // with default values. + } else if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + _, err = h.API.CreateField(r.Context(), indexName, fieldName, req.Options) + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case pilosa.ErrFieldExists: + http.Error(w, err.Error(), http.StatusConflict) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + // Encode response. + if err := json.NewEncoder(w).Encode(postFieldResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type _postFieldRequest postFieldRequest + +// Custom Unmarshal JSON to validate request body when creating a new field. If there's new FieldOptions, +// adding it to validFieldOptions to make sure the new option is validated, otherwise the request will be failed +func (p *postFieldRequest) UnmarshalJSON(b []byte) error { + // m is an overflow map used to capture additional, unexpected keys. + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return errors.Wrap(err, "unmarshaling unexpected keys") + } + + validFieldOptions := getValidOptions(pilosa.FieldOptions{}) + err := validateOptions(m, validFieldOptions) + if err != nil { + return err + } + + // Unmarshal expected values. + var _p _postFieldRequest + if err := json.Unmarshal(b, &_p); err != nil { + return errors.Wrap(err, "unmarshalling expected keys") + } + + p.Options = _p.Options + return nil + +} + +func getValidOptions(option interface{}) []string { + validOptions := []string{} + val := reflect.ValueOf(option) + for i := 0; i < val.Type().NumField(); i++ { + jsonTag := val.Type().Field(i).Tag.Get("json") + s := strings.Split(jsonTag, ",") + validOptions = append(validOptions, s[0]) + } + return validOptions +} + +type postFieldRequest struct { + Options pilosa.FieldOptions `json:"options"` +} + +type postFieldResponse struct{} + +// handleDeleteField handles DELETE /field request. +func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + err := h.API.DeleteField(r.Context(), indexName, fieldName) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } + return + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(deleteFieldResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type deleteFieldResponse struct{} + +// handlePostFieldAttrDiff handles POST /field/attr/diff requests. +func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + // Decode request. + var req postFieldAttrDiffRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + attrs, err := h.API.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks) + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrFragmentNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{ + Attrs: attrs, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type postFieldAttrDiffRequest struct { + Blocks []pilosa.AttrBlock `json:"blocks"` +} + +type postFieldAttrDiffResponse struct { + Attrs map[uint64]map[string]interface{} `json:"attrs"` +} + +// readQueryRequest parses an query parameters from r. +func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + switch r.Header.Get("Content-Type") { + case "application/x-protobuf": + return h.readProtobufQueryRequest(r) + default: + return h.readURLQueryRequest(r) + } +} + +// readProtobufQueryRequest parses query parameters in protobuf from r. +func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + // Slurp the body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } + + // Unmarshal into object. + var req internal.QueryRequest + if err := proto.Unmarshal(body, &req); err != nil { + return nil, errors.Wrap(err, "unmarshalling") + } + + return decodeQueryRequest(&req), nil +} + +// readURLQueryRequest parses query parameters from URL parameters from r. +func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + q := r.URL.Query() + + // Parse query string. + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } + query := string(buf) + + // Parse list of slices. + slices, err := parseUint64Slice(q.Get("slices")) + if err != nil { + return nil, errors.New("invalid slice argument") + } + + return &pilosa.QueryRequest{ + Query: query, + Slices: slices, + ColumnAttrs: q.Get("columnAttrs") == "true", + ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", + ExcludeColumns: q.Get("excludeColumns") == "true", + }, nil +} + +// writeQueryResponse writes the response from the executor to w. +func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *pilosa.QueryResponse) error { + if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { + return h.writeProtobufQueryResponse(w, resp) + } + return h.writeJSONQueryResponse(w, resp) +} + +// writeProtobufQueryResponse writes the response from the executor to w as protobuf. +func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { + if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil { + return errors.Wrap(err, "marshalling") + } else if _, err := w.Write(buf); err != nil { + return errors.Wrap(err, "writing") + } + return nil +} + +// writeJSONQueryResponse writes the response from the executor to w as JSON. +func (h *Handler) writeJSONQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { + return json.NewEncoder(w).Encode(resp) +} + +// handlePostImport handles /import requests. +func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Marshal into request object. + var req internal.ImportRequest + if err := proto.Unmarshal(body, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.API.Import(r.Context(), req); err != nil { + switch errors.Cause(err) { + case pilosa.ErrIndexNotFound: + fallthrough + case pilosa.ErrFieldNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case pilosa.ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Marshal response object. + buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) + if e != nil { + http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) + return + } + + // Write response. + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + w.Write(buf) +} + +// handlePostImportValue handles /import-value requests. +func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Marshal into request object. + var req internal.ImportValueRequest + if err := proto.Unmarshal(body, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err = h.API.ImportValue(r.Context(), req); err != nil { + switch errors.Cause(err) { + case pilosa.ErrIndexNotFound: + fallthrough + case pilosa.ErrFieldNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case pilosa.ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Marshal response object. + buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) + if e != nil { + http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) + return + } + + // Write response. + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + w.Write(buf) +} + +// handleGetExport handles /export requests. +func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { + switch r.Header.Get("Accept") { + case "text/csv": + h.handleGetExportCSV(w, r) + default: + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + } +} + +func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { + // Parse query parameters. + q := r.URL.Query() + index, field := q.Get("index"), q.Get("field") + + slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + if err != nil { + http.Error(w, "invalid slice", http.StatusBadRequest) + return + } + + if err = h.API.ExportCSV(r.Context(), index, field, slice, w); err != nil { + switch errors.Cause(err) { + case pilosa.ErrFragmentNotFound: + break + case pilosa.ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } +} + +// handleGetFragmentNodes handles /fragment/nodes requests. +func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + index := q.Get("index") + + // Read slice parameter. + slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + if err != nil { + http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest) + return + } + + // Retrieve fragment owner nodes. + nodes, err := h.API.SliceNodes(r.Context(), index, slice) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Write to response. + if err := json.NewEncoder(w).Encode(nodes); err != nil { + h.Logger.Printf("json write error: %s", err) + } +} + +// handleGetFragmentBlockData handles GET /fragment/block/data requests. +func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { + buf, err := h.API.FragmentBlockData(r.Context(), r.Body) + if err != nil { + if _, ok := err.(pilosa.BadRequestError); ok { + http.Error(w, err.Error(), http.StatusBadRequest) + } else if errors.Cause(err) == pilosa.ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Write response. + w.Header().Set("Content-Type", "application/protobuf") + w.Header().Set("Content-Length", strconv.Itoa(len(buf))) + w.Write(buf) +} + +// handleGetFragmentBlocks handles GET /fragment/blocks requests. +func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) { + // Read slice parameter. + q := r.URL.Query() + slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + if err != nil { + http.Error(w, "slice required", http.StatusBadRequest) + return + } + + blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), slice) + if err != nil { + if errors.Cause(err) == pilosa.ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ + Blocks: blocks, + }); err != nil { + h.Logger.Printf("block response encoding error: %s", err) + } +} + +type getFragmentBlocksResponse struct { + Blocks []pilosa.FragmentBlock `json:"blocks"` +} + +// handleGetVersion handles /version requests. +func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { + err := json.NewEncoder(w).Encode(struct { + Version string `json:"version"` + }{ + Version: h.API.Version(), + }) + if err != nil { + h.Logger.Printf("write version response error: %s", err) + } +} + +// QueryResult types. +const ( + QueryResultTypeNil uint32 = iota + QueryResultTypeRow + QueryResultTypePairs + QueryResultTypeValCount + QueryResultTypeUint64 + QueryResultTypeBool +) + +func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest { + req := &pilosa.QueryRequest{ + Query: pb.Query, + Slices: pb.Slices, + ColumnAttrs: pb.ColumnAttrs, + Remote: pb.Remote, + ExcludeRowAttrs: pb.ExcludeRowAttrs, + ExcludeColumns: pb.ExcludeColumns, + } + + return req +} + +func encodeQueryResponse(resp *pilosa.QueryResponse) *internal.QueryResponse { + pb := &internal.QueryResponse{ + Results: make([]*internal.QueryResult, len(resp.Results)), + ColumnAttrSets: pilosa.EncodeColumnAttrSets(resp.ColumnAttrSets), + } + + for i := range resp.Results { + pb.Results[i] = &internal.QueryResult{} + + switch result := resp.Results[i].(type) { + case *pilosa.Row: + pb.Results[i].Type = QueryResultTypeRow + pb.Results[i].Row = pilosa.EncodeRow(result) + case []pilosa.Pair: + pb.Results[i].Type = QueryResultTypePairs + pb.Results[i].Pairs = pilosa.EncodePairs(result) + case pilosa.ValCount: + pb.Results[i].Type = QueryResultTypeValCount + pb.Results[i].ValCount = pilosa.EncodeValCount(result) + case uint64: + pb.Results[i].Type = QueryResultTypeUint64 + pb.Results[i].N = result + case bool: + pb.Results[i].Type = QueryResultTypeBool + pb.Results[i].Changed = result + case nil: + pb.Results[i].Type = QueryResultTypeNil + } + } + + if resp.Err != nil { + pb.Err = resp.Err.Error() + } + + return pb +} + +// parseUint64Slice returns a slice of uint64s from a comma-delimited string. +func parseUint64Slice(s string) ([]uint64, error) { + var a []uint64 + for _, str := range strings.Split(s, ",") { + // Ignore blanks. + if str == "" { + continue + } + + // Parse number. + num, err := strconv.ParseUint(str, 10, 64) + if err != nil { + return nil, errors.Wrap(err, "parsing int") + } + a = append(a, num) + } + return a, nil +} + +// errorString returns the string representation of err. +func errorString(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req setCoordinatorRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest) + return + } + + oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID) + if err != nil { + if errors.Cause(err) == pilosa.ErrNodeIDNotExists { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError) + } + return + } + // Encode response. + if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ + Old: oldNode, + New: newNode, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type setCoordinatorRequest struct { + ID string `json:"id"` +} + +type setCoordinatorResponse struct { + Old *pilosa.Node `json:"old"` + New *pilosa.Node `json:"new"` +} + +// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. +func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req removeNodeRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + removeNode, err := h.API.RemoveNode(req.ID) + if err != nil { + if errors.Cause(err) == pilosa.ErrNodeIDNotExists { + http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(removeNodeResponse{ + Remove: removeNode, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type removeNodeRequest struct { + ID string `json:"id"` +} + +type removeNodeResponse struct { + Remove *pilosa.Node `json:"remove"` +} + +// handlePostClusterResizeAbort handles POST /cluster/resize/abort request. +func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { + err := h.API.ResizeAbort() + var msg string + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrNodeNotCoordinator: + http.Error(w, err.Error(), http.StatusBadRequest) + return + case pilosa.ErrResizeNotRunning: + msg = err.Error() + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + // Encode response. + if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ + Info: msg, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type clusterResizeAbortResponse struct { + Info string `json:"info"` +} + +func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { + err := h.API.RecalculateCaches(r.Context()) + if err != nil { + http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } + + err := h.API.ClusterMessage(r.Context(), r.Body) + if err != nil { + // TODO this was the previous behavior, but perhaps not everything is a bad request + http.Error(w, err.Error(), http.StatusBadRequest) + } + + if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +func (h *Handler) GetAPI() *pilosa.API { + return h.API +} + +type defaultClusterMessageResponse struct{} + +type queryValidationSpec struct { + required []string + args map[string]struct{} +} + +func queryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { + args := map[string]struct{}{} + for _, arg := range requiredArgs { + args[arg] = struct{}{} + } + + return &queryValidationSpec{ + required: requiredArgs, + args: args, + } +} + +func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { + for _, arg := range args { + s.args[arg] = struct{}{} + } + return s +} + +func (s queryValidationSpec) validate(query url.Values) error { + for _, req := range s.required { + if query.Get(req) == "" { + return errors.Errorf("%s is required", req) + } + } + for k := range query { + if _, ok := s.args[k]; !ok { + return errors.Errorf("%s is not a valid argument", k) + } + } + return nil +} + +func GetHTTPClient(t *tls.Config) *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 200, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + if t != nil { + transport.TLSClientConfig = t + } + return &http.Client{Transport: transport} +} diff --git a/handler_internal_test.go b/http/handler_internal_test.go similarity index 94% rename from handler_internal_test.go rename to http/handler_internal_test.go index 837c456ca..fa7c23060 100644 --- a/handler_internal_test.go +++ b/http/handler_internal_test.go @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa +package http import ( "encoding/json" "reflect" "testing" + + "github.com/pilosa/pilosa" ) // Test custom UnmarshalJSON for postIndexRequest object @@ -27,7 +29,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { expected postIndexRequest err string }{ - {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}}, + {json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, @@ -62,12 +64,12 @@ func TestPostFieldRequestUnmarshalJSON(t *testing.T) { expected postFieldRequest err string }{ - {json: `{"options": {}}`, expected: postFieldRequest{Options: FieldOptions{}}}, + {json: `{"options": {}}`, expected: postFieldRequest{Options: pilosa.FieldOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, {json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"}, - {json: `{"options": {"cacheType": "type"}}`, expected: postFieldRequest{Options: FieldOptions{CacheType: "type"}}}, + {json: `{"options": {"cacheType": "type"}}`, expected: postFieldRequest{Options: pilosa.FieldOptions{CacheType: "type"}}}, {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, } for _, test := range tests { diff --git a/handler_test.go b/http/handler_test.go similarity index 93% rename from handler_test.go rename to http/handler_test.go index f9e490d9d..39941a669 100644 --- a/handler_test.go +++ b/http/handler_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package http_test import ( "bytes" @@ -21,7 +21,7 @@ import ( "fmt" "io" "io/ioutil" - "net/http" + gohttp "net/http" "net/http/httptest" "reflect" "strings" @@ -29,6 +29,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" @@ -49,7 +50,7 @@ func TestHandlerPanics(t *testing.T) { if !bytes.Contains(bufbytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) { t.Fatalf("expected panic in log, but got: %s", bufbytes) } - if w.Code != http.StatusInternalServerError { + if w.Code != gohttp.StatusInternalServerError { t.Fatalf("expected internal server error, but got: %v", w.Code) } bodyBytes := w.Body.Bytes() @@ -69,7 +70,7 @@ func TestHandler_NotFound(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) - if w.Code != http.StatusNotFound { + if w.Code != gohttp.StatusNotFound { t.Fatalf("invalid status: %d", w.Code) } } @@ -101,7 +102,7 @@ func TestHandler_Schema(t *testing.T) { h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { @@ -142,7 +143,7 @@ func TestHandler_Status(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}],"localID":"node0"}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -156,9 +157,9 @@ func TestHandler_Info(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", SliceWidth) { + } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) { t.Fatalf("unexpected body: %s", body) } } @@ -173,7 +174,7 @@ func TestHandler_ClusterResizeAbort(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { bod, err := ioutil.ReadAll(w.Body) t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) } else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" { @@ -188,20 +189,20 @@ func TestHandler_MaxSlices(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*pilosa.SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*pilosa.SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*pilosa.SliceWidth)+4) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*pilosa.SliceWidth)+8) h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -229,7 +230,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -270,7 +271,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, req) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } } @@ -286,7 +287,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { h.API.Holder = hldr.Holder h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -295,7 +296,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { func TestHandler_Query_Params_Err(t *testing.T) { w := httptest.NewRecorder() test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -317,7 +318,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -340,14 +341,14 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeUint64 { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if n := resp.Results[0].N; n != 100 { t.Fatalf("unexpected n: %d", n) @@ -370,7 +371,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -403,7 +404,7 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -428,16 +429,16 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { + } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) @@ -486,7 +487,7 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { r.Header.Set("Content-Type", "application/x-protobuf") r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } @@ -494,9 +495,9 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { + if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) @@ -536,7 +537,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -562,14 +563,14 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypePairs { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if a := resp.Results[0].GetPairs(); len(a) != 2 { t.Fatalf("unexpected pair length: %d", len(a)) @@ -590,7 +591,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"executing: marker"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -613,7 +614,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } @@ -635,7 +636,7 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) - if w.Code != http.StatusMethodNotAllowed { + if w.Code != gohttp.StatusMethodNotAllowed { t.Fatalf("invalid status: %d", w.Code) } } @@ -650,7 +651,7 @@ func TestHandler_Query_ErrParse(t *testing.T) { h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -672,14 +673,14 @@ func TestHandler_Index_Delete(t *testing.T) { } // Send request to delete index. - resp, err := http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) + resp, err := gohttp.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) if err != nil { t.Fatal(err) } defer resp.Body.Close() // Verify body response. - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != gohttp.StatusOK { t.Fatalf("unexpected status: %d", resp.StatusCode) } else if buf, err := ioutil.ReadAll(resp.Body); err != nil { t.Fatal(err) @@ -707,7 +708,7 @@ func TestHandler_DeleteField(t *testing.T) { h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/field/f1", strings.NewReader(""))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -749,7 +750,7 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") // Send block checksums to determine diff. - resp, err := http.Post( + resp, err := gohttp.Post( s.URL+"/index/i/attr/diff", "application/json", strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), @@ -799,7 +800,7 @@ func TestHandler_Field_AttrStore_Diff(t *testing.T) { blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") // Send block checksums to determine diff. - resp, err := http.Post( + resp, err := gohttp.Post( s.URL+"/index/i/field/meta/attr/diff", "application/json", strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), @@ -831,7 +832,7 @@ func TestHandler_Version(t *testing.T) { if strings.HasPrefix(version, "v") { version = version[1:] } - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if w.Body.String() != `{"version":"`+version+`"}`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) @@ -851,7 +852,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -861,7 +862,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { w = httptest.NewRecorder() r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } @@ -869,7 +870,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { w = httptest.NewRecorder() r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } } @@ -885,7 +886,7 @@ func TestHandler_Expvars(t *testing.T) { w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } } @@ -908,7 +909,7 @@ func TestHandler_RecalculateCaches(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) - if w.Code != http.StatusNoContent { + if w.Code != gohttp.StatusNoContent { t.Fatalf("unexpected status code: %d", w.Code) } @@ -939,7 +940,7 @@ func TestHandler_CORS(t *testing.T) { } // CORS config should allow preflight response - handler = test.MustNewHandler(pilosa.OptHandlerAllowedOrigins([]string{"http://test/"})) + handler = test.MustNewHandler(http.OptHandlerAllowedOrigins([]string{"http://test/"})) w = httptest.NewRecorder() handler.ServeHTTP(w, req) result = w.Result() diff --git a/pilosa.go b/pilosa.go index 6a7e74dfb..a87bab4fc 100644 --- a/pilosa.go +++ b/pilosa.go @@ -88,17 +88,17 @@ type ColumnAttrSet struct { Attrs map[string]interface{} `json:"attrs,omitempty"` } -// encodeColumnAttrSets converts a into its internal representation. -func encodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { +// EncodeColumnAttrSets converts a into its internal representation. +func EncodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { other := make([]*internal.ColumnAttrSet, len(a)) for i := range a { - other[i] = encodeColumnAttrSet(a[i]) + other[i] = EncodeColumnAttrSet(a[i]) } return other } -// encodeColumnAttrSet converts set into its internal representation. -func encodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { +// EncodeColumnAttrSet converts set into its internal representation. +func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { return &internal.ColumnAttrSet{ ID: set.ID, Attrs: encodeAttrs(set.Attrs), diff --git a/row.go b/row.go index ec2c42726..b7643a975 100644 --- a/row.go +++ b/row.go @@ -261,8 +261,8 @@ func (r *Row) Columns() []uint64 { return a } -// encodeRow converts r into its internal representation. -func encodeRow(r *Row) *internal.Row { +// EncodeRow converts r into its internal representation. +func EncodeRow(r *Row) *internal.Row { if r == nil { return nil } @@ -273,8 +273,8 @@ func encodeRow(r *Row) *internal.Row { } } -// decodeRow converts r from its internal representation. -func decodeRow(pr *internal.Row) *Row { +// DecodeRow converts r from its internal representation. +func DecodeRow(pr *internal.Row) *Row { if pr == nil { return nil } diff --git a/server.go b/server.go index 471319dd7..a324ebd49 100644 --- a/server.go +++ b/server.go @@ -59,7 +59,7 @@ type Server struct { executor *Executor // External - handler *Handler + handler Handlerer Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver Gossiper Gossiper @@ -127,7 +127,7 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption { } } -func OptServerHandler(h *Handler) ServerOption { +func OptServerHandler(h Handlerer) ServerOption { return func(s *Server) error { s.handler = h return nil @@ -162,16 +162,24 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { } } +// TODO: Remove RemoteClient func OptServerRemoteClient(c *http.Client) ServerOption { return func(s *Server) error { - s.executor = NewExecutor(c) s.remoteClient = c - s.defaultClient = NewInternalHTTPClientFromURI(nil, c) s.Cluster.RemoteClient = c return nil } } +func OptServerInternalClient(c InternalClient) ServerOption { + return func(s *Server) error { + s.executor = NewExecutor(ExecutorOptInternalQueryClient(c)) + s.defaultClient = c + s.Cluster.InternalClient = c + return nil + } +} + func OptServerStatsClient(sc StatsClient) ServerOption { return func(s *Server) error { s.Holder.Stats = sc @@ -203,15 +211,15 @@ func OptServerURI(uri *URI) ServerOption { // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { - handler, err := NewHandler() - if err != nil { - return nil, errors.Wrap(err, "initializing handler") - } + //handler, err := NewNopHandler() + //if err != nil { + // return nil, errors.Wrap(err, "initializing handler") + //} s := &Server{ - closing: make(chan struct{}), - Cluster: NewCluster(), - Holder: NewHolder(), - handler: handler, + closing: make(chan struct{}), + Cluster: NewCluster(), + Holder: NewHolder(), + //handler: handler, Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), @@ -268,7 +276,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.executor.Node = node s.executor.Cluster = s.Cluster s.executor.MaxWritesPerRequest = s.maxWritesPerRequest - s.handler.API.Executor = s.executor + s.handler.GetAPI().Executor = s.executor return s, nil } @@ -300,11 +308,12 @@ func (s *Server) Open() error { s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest // Initialize HTTP handler. - s.handler.API.Holder = s.Holder - s.handler.API.Broadcaster = s.Broadcaster - s.handler.API.BroadcastHandler = s - s.handler.API.StatusHandler = s - s.handler.API.Cluster = s.Cluster + api := s.handler.GetAPI() + api.Holder = s.Holder + api.Broadcaster = s.Broadcaster + api.BroadcastHandler = s + api.StatusHandler = s + api.Cluster = s.Cluster // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster diff --git a/server/server.go b/server/server.go index bb75a1d2e..febb79529 100644 --- a/server/server.go +++ b/server/server.go @@ -25,7 +25,6 @@ import ( "log" "math/rand" "net" - "net/http" "os" "os/signal" "strconv" @@ -39,6 +38,7 @@ import ( "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gopsutil" "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/statsd" "github.com/pkg/errors" ) @@ -164,13 +164,17 @@ func (m *Command) SetupServer() error { } m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime) - handler, err := pilosa.NewHandler(pilosa.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins)) + api := pilosa.NewAPI() + api.Logger = m.logger + + handler, err := http.NewHandler( + http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), + http.OptHandlerAPI(api), + http.OptHandlerLogger(m.logger), + ) if err != nil { return errors.Wrap(err, "wrapping handler") } - handler.Logger = m.logger - handler.API = pilosa.NewAPI() - handler.API.Logger = m.logger uri, err := pilosa.AddressWithDefaults(m.Config.Bind) if err != nil { @@ -211,8 +215,8 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "getting listener") } - c := GetHTTPClient(TLSConfig) - handler.API.RemoteClient = c + c := http.GetHTTPClient(TLSConfig) + api.RemoteClient = c m.Server, err = pilosa.NewServer( pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), @@ -232,31 +236,12 @@ func (m *Command) SetupServer() error { pilosa.OptServerListener(ln), pilosa.OptServerURI(uri), pilosa.OptServerRemoteClient(c), + pilosa.OptServerInternalClient(http.NewInternalHTTPClientFromURI(uri, c)), ) return errors.Wrap(err, "new server") } -func GetHTTPClient(t *tls.Config) *http.Client { - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 1000, - MaxIdleConnsPerHost: 200, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - if t != nil { - transport.TLSClientConfig = t - } - return &http.Client{Transport: transport} -} - // SetupNetworking sets up internode communication based on the configuration. func (m *Command) SetupNetworking() error { diff --git a/server/server_test.go b/server/server_test.go index 4356ad241..32d57b01c 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -29,6 +29,7 @@ import ( "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -44,7 +45,7 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) + client, err := http.NewInternalHTTPClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } diff --git a/test/client.go b/test/client.go index 9e391e88b..faef0b480 100644 --- a/test/client.go +++ b/test/client.go @@ -15,19 +15,19 @@ package test import ( - "net/http" + gohttp "net/http" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" ) // Client represents a test wrapper for pilosa.Client. type Client struct { - *pilosa.InternalHTTPClient + *http.InternalHTTPClient } // MustNewClient returns a new instance of Client. Panic on error. -func MustNewClient(host string, h *http.Client) *Client { - c, err := pilosa.NewInternalHTTPClient(host, h) +func MustNewClient(host string, h *gohttp.Client) *Client { + c, err := http.NewInternalHTTPClient(host, h) if err != nil { panic(err) } diff --git a/test/executor.go b/test/executor.go index 8cd6391c3..2e2912d60 100644 --- a/test/executor.go +++ b/test/executor.go @@ -15,12 +15,12 @@ package test import ( - "net/http" + gohttp "net/http" "strings" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" ) // Executor represents a test wrapper for pilosa.Executor. @@ -28,16 +28,17 @@ type Executor struct { *pilosa.Executor } -var remoteClient *http.Client +var remoteClient *gohttp.Client func init() { - remoteClient = server.GetHTTPClient(nil) + remoteClient = http.GetHTTPClient(nil) } // NewExecutor returns a new instance of Executor. // The executor always matches the uri of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - executor := pilosa.NewExecutor(remoteClient) + client := http.NewInternalHTTPClientFromURI(nil, remoteClient) + executor := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(client)) e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster diff --git a/test/handler.go b/test/handler.go index 401fa64ab..27fa20503 100644 --- a/test/handler.go +++ b/test/handler.go @@ -19,25 +19,26 @@ import ( "encoding/json" "io" "io/ioutil" - "net/http" + gohttp "net/http" "net/http/httptest" "net/url" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) // Handler represents a test wrapper for pilosa.Handler. type Handler struct { - *pilosa.Handler + *http.Handler Executor HandlerExecutor } // NewHandler returns a new instance of Handler. -func NewHandler(opts ...pilosa.HandlerOption) (*Handler, error) { - handler, err := pilosa.NewHandler(opts...) +func NewHandler(opts ...http.HandlerOption) (*Handler, error) { + handler, err := http.NewHandler(opts...) if err != nil { return nil, err } @@ -55,7 +56,7 @@ func NewHandler(opts ...pilosa.HandlerOption) (*Handler, error) { } // MustNewHandler returns a new instance of Handler. -func MustNewHandler(opts ...pilosa.HandlerOption) *Handler { +func MustNewHandler(opts ...http.HandlerOption) *Handler { h, err := NewHandler(opts...) if err != nil { panic(err) @@ -145,8 +146,8 @@ func MustParseURLHost(rawurl string) string { } // MustNewHTTPRequest creates a new HTTP request. Panic on error. -func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request { - req, err := http.NewRequest(method, urlStr, body) +func MustNewHTTPRequest(method, urlStr string, body io.Reader) *gohttp.Request { + req, err := gohttp.NewRequest(method, urlStr, body) if err != nil { panic(err) } diff --git a/test/pilosa.go b/test/pilosa.go index 4a741ea8b..426988f3f 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -19,15 +19,15 @@ import ( "fmt" "io" "io/ioutil" - "net/http" + gohttp "net/http" "os" "strings" "testing" "time" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" @@ -238,8 +238,8 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } // Client returns a client to connect to the program. -func (m *Main) Client() *pilosa.InternalHTTPClient { - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) +func (m *Main) Client() *http.InternalHTTPClient { + client, err := http.NewInternalHTTPClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { panic(err) } @@ -249,7 +249,7 @@ func (m *Main) Client() *pilosa.InternalHTTPClient { // Query executes a query against the program through the HTTP API. func (m *Main) Query(index, rawQuery, query string) (string, error) { resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query) - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != gohttp.StatusOK { return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) } return resp.Body, nil @@ -267,11 +267,11 @@ func (m *Main) RecalculateCaches() error { // MustDo executes http.Do() with an http.NewRequest(). Panic on error. func MustDo(method, urlStr string, body string) *httpResponse { - req, err := http.NewRequest(method, urlStr, strings.NewReader(body)) + req, err := gohttp.NewRequest(method, urlStr, strings.NewReader(body)) if err != nil { panic(err) } - resp, err := http.DefaultClient.Do(req) + resp, err := gohttp.DefaultClient.Do(req) if err != nil { panic(err) } @@ -287,6 +287,6 @@ func MustDo(method, urlStr string, body string) *httpResponse { // httpResponse is a wrapper for http.Response that holds the Body as a string. type httpResponse struct { - *http.Response + *gohttp.Response Body string } From 64287905bd38e578b1c5c8ad5738db73e88d4dcb Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 13 Jun 2018 08:56:14 -0500 Subject: [PATCH 60/78] Add missing argument to Errorf call --- server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index febb79529..ba2d00635 100644 --- a/server/server.go +++ b/server/server.go @@ -338,7 +338,7 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { case "nop", "none": return pilosa.NopStatsClient, nil default: - return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].") + return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].", name) } } From daffa3b12515b78025a6a658aad7facc659b8496 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 13 Jun 2018 09:04:04 -0500 Subject: [PATCH 61/78] Rename InternalHTTPClient -> InternalClient --- ctl/common.go | 4 +-- holder_test.go | 2 +- http/client.go | 64 +++++++++++++++++++++---------------------- http/client_test.go | 6 ++-- server/server.go | 2 +- server/server_test.go | 2 +- test/client.go | 6 ++-- test/executor.go | 2 +- test/pilosa.go | 4 +-- 9 files changed, 46 insertions(+), 46 deletions(-) diff --git a/ctl/common.go b/ctl/common.go index fe0c46f7b..44e01d406 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -37,7 +37,7 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP } // CommandClient returns a pilosa.InternalHTTPClient for the command -func CommandClient(cmd CommandWithTLSSupport) (*http.InternalHTTPClient, error) { +func CommandClient(cmd CommandWithTLSSupport) (*http.InternalClient, error) { tlsConfig := cmd.TLSConfiguration() var TLSConfig *tls.Config if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" { @@ -50,7 +50,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*http.InternalHTTPClient, error) InsecureSkipVerify: tlsConfig.SkipVerify, } } - client, err := http.NewInternalHTTPClient(cmd.TLSHost(), http.GetHTTPClient(TLSConfig)) + client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(TLSConfig)) if err != nil { return nil, errors.Wrap(err, "getting internal client") } diff --git a/holder_test.go b/holder_test.go index b47e019eb..4375d970f 100644 --- a/holder_test.go +++ b/holder_test.go @@ -360,7 +360,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { cluster := test.NewCluster(2) client := http.GetHTTPClient(nil) - httpClient := http.NewInternalHTTPClientFromURI(uri, client) + httpClient := http.NewInternalClientFromURI(uri, client) cluster.InternalClient = httpClient cluster.RemoteClient = client diff --git a/http/client.go b/http/client.go index b3ffb20cb..d8f0d290d 100644 --- a/http/client.go +++ b/http/client.go @@ -40,16 +40,16 @@ type ClientOptions struct { TLS *tls.Config } -// InternalHTTPClient represents a client to the Pilosa cluster. -type InternalHTTPClient struct { +// InternalClient represents a client to the Pilosa cluster. +type InternalClient struct { defaultURI *pilosa.URI // The client to use for HTTP communication. HTTPClient *http.Client } -// NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host. -func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTTPClient, error) { +// NewInternalClient returns a new instance of InternalClient to connect to host. +func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, error) { if host == "" { return nil, pilosa.ErrHostRequired } @@ -59,27 +59,27 @@ func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTT return nil, errors.Wrap(err, "getting URI") } - client := NewInternalHTTPClientFromURI(uri, remoteClient) + client := NewInternalClientFromURI(uri, remoteClient) return client, nil } -func NewInternalHTTPClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalHTTPClient { - return &InternalHTTPClient{ +func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient { + return &InternalClient{ defaultURI: defaultURI, HTTPClient: remoteClient, } } // Host returns the host the client was initialized with. -func (c *InternalHTTPClient) Host() *pilosa.URI { return c.defaultURI } +func (c *InternalClient) Host() *pilosa.URI { return c.defaultURI } // MaxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { +func (c *InternalClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { return c.maxSliceByIndex(ctx) } // maxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { +func (c *InternalClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { // Execute request against the host. u := uriPathToURL(c.defaultURI, "/slices/max") @@ -109,7 +109,7 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]ui } // Schema returns all index and field schema information. -func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) { +func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) { // Execute request against the host. u := c.defaultURI.Path("/schema") @@ -138,7 +138,7 @@ func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, e } // CreateIndex creates a new index on the server. -func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error { +func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error { // Encode query request. buf, err := json.Marshal(&postIndexRequest{ Options: opt, @@ -183,7 +183,7 @@ func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt } // FragmentNodes returns a list of nodes that own a slice. -func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*pilosa.Node, error) { +func (c *InternalClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*pilosa.Node, error) { // Execute request against the host. u := uriPathToURL(c.defaultURI, "/fragment/nodes") u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() @@ -214,12 +214,12 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl } // Query executes query against the index. -func (c *InternalHTTPClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { return c.QueryNode(ctx, c.defaultURI, index, queryRequest) } // QueryNode executes query against the index, sending the request to the node specified. -func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { if index == "" { return nil, pilosa.ErrIndexRequired } else if queryRequest.Query == "" { @@ -270,7 +270,7 @@ func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *pilosa.URI, ind } // Import bulk imports bits for a single slice to a host. -func (c *InternalHTTPClient) Import(ctx context.Context, index, field string, slice uint64, bits []pilosa.Bit) error { +func (c *InternalClient) Import(ctx context.Context, index, field string, slice uint64, bits []pilosa.Bit) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -299,7 +299,7 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, field string, sl } // ImportK bulk imports bits specified by string keys to a host. -func (c *InternalHTTPClient) ImportK(ctx context.Context, index, field string, columns []pilosa.Bit) error { +func (c *InternalClient) ImportK(ctx context.Context, index, field string, columns []pilosa.Bit) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -323,7 +323,7 @@ func (c *InternalHTTPClient) ImportK(ctx context.Context, index, field string, c return nil } -func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, options pilosa.IndexOptions) error { +func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options pilosa.IndexOptions) error { err := c.CreateIndex(ctx, name, options) if err == nil || err == pilosa.ErrIndexExists { return nil @@ -331,7 +331,7 @@ func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, optio return err } -func (c *InternalHTTPClient) EnsureField(ctx context.Context, indexName string, fieldName string, options pilosa.FieldOptions) error { +func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string, options pilosa.FieldOptions) error { err := c.CreateField(ctx, indexName, fieldName, options) if err == nil || err == pilosa.ErrFieldExists { return nil @@ -383,7 +383,7 @@ func marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, erro } // importNode sends a pre-marshaled import request to a node. -func (c *InternalHTTPClient) importNode(ctx context.Context, node *pilosa.Node, buf []byte) error { +func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, buf []byte) error { // Create URL & HTTP request. u := nodePathToURL(node, "/import") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) @@ -421,7 +421,7 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *pilosa.Node, } // ImportValue bulk imports field values for a single slice to a host. -func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []pilosa.FieldValue) error { +func (c *InternalClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []pilosa.FieldValue) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -470,7 +470,7 @@ func marshalImportValuePayload(index, field string, slice uint64, vals []pilosa. } // importValueNode sends a pre-marshaled import request to a node. -func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *pilosa.Node, buf []byte) error { +func (c *InternalClient) importValueNode(ctx context.Context, node *pilosa.Node, buf []byte) error { // Create URL & HTTP request. u := nodePathToURL(node, "/import-value") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) @@ -508,7 +508,7 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *pilosa.N } // ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { +func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -538,7 +538,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, field string, } // exportNode copies a CSV export from a node to w. -func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, slice uint64, w io.Writer) error { +func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, slice uint64, w io.Writer) error { // Create URL. u := nodePathToURL(node, "/export") u.RawQuery = url.Values{ @@ -575,14 +575,14 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *pilosa.Nod return nil } -func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri pilosa.URI) (io.ReadCloser, error) { +func (c *InternalClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri pilosa.URI) (io.ReadCloser, error) { node := &pilosa.Node{ URI: uri, } return c.backupSliceNode(ctx, index, field, slice, node) } -func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *pilosa.Node) (io.ReadCloser, error) { +func (c *InternalClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *pilosa.Node) (io.ReadCloser, error) { u := nodePathToURL(node, "/fragment/data") u.RawQuery = url.Values{ "index": {index}, @@ -617,7 +617,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, field s } // CreateField creates a new field on the server. -func (c *InternalHTTPClient) CreateField(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { +func (c *InternalClient) CreateField(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { if index == "" { return pilosa.ErrIndexRequired } @@ -667,7 +667,7 @@ func (c *InternalHTTPClient) CreateField(ctx context.Context, index, field strin // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64) ([]pilosa.FragmentBlock, error) { +func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64) ([]pilosa.FragmentBlock, error) { if uri == nil { uri = c.defaultURI } @@ -711,7 +711,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI } // BlockData returns row/column id pairs for a block. -func (c *InternalHTTPClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { +func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ Index: index, Field: field, @@ -758,7 +758,7 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, uri *pilosa.URI, ind } // ColumnAttrDiff returns data from differing blocks on a remote host. -func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { +func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { if uri == nil { uri = c.defaultURI } @@ -801,7 +801,7 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI } // RowAttrDiff returns data from differing blocks on a remote host. -func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { +func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { if uri == nil { uri = c.defaultURI } @@ -846,7 +846,7 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, i } // SendMessage posts a message synchronously. -func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb proto.Message) error { +func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) if err != nil { return fmt.Errorf("marshaling message: %v", err) diff --git a/http/client_test.go b/http/client_test.go index 7920d03cf..3853d2335 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -61,7 +61,7 @@ func TestClient_MultiNode(t *testing.T) { } s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - httpClient := http.NewInternalHTTPClientFromURI(&cluster.Nodes[0].URI, defaultClient) + httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) e.Holder = hldr[0].Holder e.Node = cluster.Nodes[0] @@ -69,7 +69,7 @@ func TestClient_MultiNode(t *testing.T) { return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - httpClient := http.NewInternalHTTPClientFromURI(&cluster.Nodes[0].URI, defaultClient) + httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) e.Holder = hldr[1].Holder e.Node = cluster.Nodes[1] @@ -77,7 +77,7 @@ func TestClient_MultiNode(t *testing.T) { return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - httpClient := http.NewInternalHTTPClientFromURI(&cluster.Nodes[0].URI, defaultClient) + httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) e.Holder = hldr[2].Holder e.Node = cluster.Nodes[2] diff --git a/server/server.go b/server/server.go index ba2d00635..42b547d43 100644 --- a/server/server.go +++ b/server/server.go @@ -236,7 +236,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerListener(ln), pilosa.OptServerURI(uri), pilosa.OptServerRemoteClient(c), - pilosa.OptServerInternalClient(http.NewInternalHTTPClientFromURI(uri, c)), + pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), ) return errors.Wrap(err, "new server") diff --git a/server/server_test.go b/server/server_test.go index 32d57b01c..a8cc2f8f4 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -45,7 +45,7 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := http.NewInternalHTTPClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) + client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } diff --git a/test/client.go b/test/client.go index faef0b480..34d5e17d0 100644 --- a/test/client.go +++ b/test/client.go @@ -22,14 +22,14 @@ import ( // Client represents a test wrapper for pilosa.Client. type Client struct { - *http.InternalHTTPClient + *http.InternalClient } // MustNewClient returns a new instance of Client. Panic on error. func MustNewClient(host string, h *gohttp.Client) *Client { - c, err := http.NewInternalHTTPClient(host, h) + c, err := http.NewInternalClient(host, h) if err != nil { panic(err) } - return &Client{InternalHTTPClient: c} + return &Client{InternalClient: c} } diff --git a/test/executor.go b/test/executor.go index 2e2912d60..ae0a412d5 100644 --- a/test/executor.go +++ b/test/executor.go @@ -37,7 +37,7 @@ func init() { // NewExecutor returns a new instance of Executor. // The executor always matches the uri of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - client := http.NewInternalHTTPClientFromURI(nil, remoteClient) + client := http.NewInternalClientFromURI(nil, remoteClient) executor := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(client)) e := &Executor{Executor: executor} e.Holder = holder diff --git a/test/pilosa.go b/test/pilosa.go index 426988f3f..dca6243fb 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -238,8 +238,8 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } // Client returns a client to connect to the program. -func (m *Main) Client() *http.InternalHTTPClient { - client, err := http.NewInternalHTTPClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) +func (m *Main) Client() *http.InternalClient { + client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { panic(err) } From ecbbd31b4ebb53cafae467ebb33001e4e95fe8da Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 13 Jun 2018 09:18:50 -0500 Subject: [PATCH 62/78] Improve naming --- client.go | 2 +- executor.go | 7 ++++--- holder_test.go | 2 +- http/client_test.go | 6 +++--- server.go | 2 +- test/executor.go | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/client.go b/client.go index 56cba08e1..b5dbe809a 100644 --- a/client.go +++ b/client.go @@ -18,7 +18,7 @@ type Bit struct { Timestamp int64 } -// FieldValues represents the value for a column within a +// FieldValue represents the value for a column within a // range-encoded field. type FieldValue struct { ColumnID uint64 diff --git a/executor.go b/executor.go index 25fc98ac9..17fef6165 100644 --- a/executor.go +++ b/executor.go @@ -52,9 +52,10 @@ type Executor struct { MaxWritesPerRequest int } -type ExecutorOpt func(e *Executor) error +// ExecutorOption is a functional option type for pilosa.Executor +type ExecutorOption func(e *Executor) error -func ExecutorOptInternalQueryClient(c InternalQueryClient) ExecutorOpt { +func OptExecutorInternalQueryClient(c InternalQueryClient) ExecutorOption { return func(e *Executor) error { e.client = c return nil @@ -62,7 +63,7 @@ func ExecutorOptInternalQueryClient(c InternalQueryClient) ExecutorOpt { } // NewExecutor returns a new instance of Executor. -func NewExecutor(opts ...ExecutorOpt) *Executor { +func NewExecutor(opts ...ExecutorOption) *Executor { e := &Executor{ client: NewNopInternalQueryClient(), } diff --git a/holder_test.go b/holder_test.go index 4375d970f..b56863bb0 100644 --- a/holder_test.go +++ b/holder_test.go @@ -373,7 +373,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { defer hldr1.Close() s.Handler.API.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) + e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr1.Holder e.Node = cluster.Nodes[1] e.Cluster = cluster diff --git a/http/client_test.go b/http/client_test.go index 3853d2335..1cc972ff6 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -62,7 +62,7 @@ func TestClient_MultiNode(t *testing.T) { s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) - e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) + e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[0].Holder e.Node = cluster.Nodes[0] e.Cluster = cluster @@ -70,7 +70,7 @@ func TestClient_MultiNode(t *testing.T) { } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) - e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) + e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[1].Holder e.Node = cluster.Nodes[1] e.Cluster = cluster @@ -78,7 +78,7 @@ func TestClient_MultiNode(t *testing.T) { } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) - e := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(httpClient)) + e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[2].Holder e.Node = cluster.Nodes[2] e.Cluster = cluster diff --git a/server.go b/server.go index a324ebd49..d73c9ce82 100644 --- a/server.go +++ b/server.go @@ -173,7 +173,7 @@ func OptServerRemoteClient(c *http.Client) ServerOption { func OptServerInternalClient(c InternalClient) ServerOption { return func(s *Server) error { - s.executor = NewExecutor(ExecutorOptInternalQueryClient(c)) + s.executor = NewExecutor(OptExecutorInternalQueryClient(c)) s.defaultClient = c s.Cluster.InternalClient = c return nil diff --git a/test/executor.go b/test/executor.go index ae0a412d5..c04f91eca 100644 --- a/test/executor.go +++ b/test/executor.go @@ -38,7 +38,7 @@ func init() { // The executor always matches the uri of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { client := http.NewInternalClientFromURI(nil, remoteClient) - executor := pilosa.NewExecutor(pilosa.ExecutorOptInternalQueryClient(client)) + executor := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(client)) e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster From 952db994c7c8217ccfbed64ea045c8211c7de395 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 13 Jun 2018 09:21:15 -0500 Subject: [PATCH 63/78] Fix mistake in interface name --- handler.go | 4 ++-- server.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/handler.go b/handler.go index e70368c49..f9c3435af 100644 --- a/handler.go +++ b/handler.go @@ -60,7 +60,7 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { return json.Marshal(output) } -type Handlerer interface { +type Handler interface { http.Handler GetAPI() *API } @@ -73,6 +73,6 @@ func (n *NopHandler) GetAPI() *API { return nil } -func NewNopHandler() Handlerer { +func NewNopHandler() Handler { return &NopHandler{} } diff --git a/server.go b/server.go index d73c9ce82..98a2dc16b 100644 --- a/server.go +++ b/server.go @@ -59,7 +59,7 @@ type Server struct { executor *Executor // External - handler Handlerer + handler Handler Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver Gossiper Gossiper @@ -127,7 +127,7 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption { } } -func OptServerHandler(h Handlerer) ServerOption { +func OptServerHandler(h Handler) ServerOption { return func(s *Server) error { s.handler = h return nil From 37153daf158ad711aad49093745f0d805c315242 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 13 Jun 2018 09:21:25 -0500 Subject: [PATCH 64/78] Remove commented code --- server.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/server.go b/server.go index 98a2dc16b..b1f0f3d10 100644 --- a/server.go +++ b/server.go @@ -211,15 +211,10 @@ func OptServerURI(uri *URI) ServerOption { // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { - //handler, err := NewNopHandler() - //if err != nil { - // return nil, errors.Wrap(err, "initializing handler") - //} s := &Server{ - closing: make(chan struct{}), - Cluster: NewCluster(), - Holder: NewHolder(), - //handler: handler, + closing: make(chan struct{}), + Cluster: NewCluster(), + Holder: NewHolder(), Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), From 8021fc389b241f5c4941305dfdba0839358e4685 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 8 Jun 2018 15:48:23 -0500 Subject: [PATCH 65/78] un-export (some) Cluster methods --- api.go | 18 +- cluster.go | 422 +++++++++----------- cluster_internal_test.go | 470 +++++++++++++++++++++- cluster_test.go | 494 ------------------------ executor.go | 8 +- executor_test.go | 66 ++-- fragment.go | 4 +- holder.go | 4 +- holder_test.go | 6 +- http/client_test.go | 11 +- server.go | 22 +- stats_test.go | 8 +- test/cluster.go | 378 +----------------- utils_test.go => utils_internal_test.go | 16 +- 14 files changed, 753 insertions(+), 1174 deletions(-) delete mode 100644 cluster_test.go rename utils_test.go => utils_internal_test.go (97%) diff --git a/api.go b/api.go index 3edb1b915..36af91b0f 100644 --- a/api.go +++ b/api.go @@ -291,7 +291,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin } // Validate that this handler owns the slice. - if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { + if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) return ErrClusterDoesNotOwnSlice } @@ -327,7 +327,7 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) return nil, errors.Wrap(err, "validating api method") } - return api.Cluster.SliceNodes(indexName, slice), nil + return api.Cluster.sliceNodes(indexName, slice), nil } // MarshalFragment returns an object which can write the specified fragment's data @@ -681,7 +681,7 @@ func (api *API) LongQueryTime() time.Duration { func (api *API) indexField(indexName string, fieldName string, slice uint64) (*Index, *Field, error) { // Validate that this handler owns the slice. - if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { + if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) return nil, nil, ErrClusterDoesNotOwnSlice } @@ -709,15 +709,15 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode return nil, nil, errors.Wrap(err, "validating api method") } - oldNode = api.Cluster.NodeByID(api.Cluster.Coordinator) - newNode = api.Cluster.NodeByID(id) + oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) + newNode = api.Cluster.nodeByID(id) if newNode == nil { return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node") } // If the new coordinator is this node, do the SetCoordinator directly. if newNode.ID == api.LocalID() { - return oldNode, newNode, api.Cluster.SetCoordinator(newNode) + return oldNode, newNode, api.Cluster.setCoordinator(newNode) } // Send the set-coordinator message to new node. @@ -739,13 +739,13 @@ func (api *API) RemoveNode(id string) (*Node, error) { return nil, errors.Wrap(err, "validating api method") } - removeNode := api.Cluster.nodeByID(id) + removeNode := api.Cluster.unprotectedNodeByID(id) if removeNode == nil { return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") } // Start the resize process (similar to NodeJoin) - err := api.Cluster.NodeLeave(removeNode) + err := api.Cluster.nodeLeave(removeNode) if err != nil { return removeNode, errors.Wrap(err, "calling node leave") } @@ -758,7 +758,7 @@ func (api *API) ResizeAbort() error { return errors.Wrap(err, "validating api method") } - err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted) + err := api.Cluster.completeCurrentJob(resizeJobStateAborted) return errors.Wrap(err, "complete current job") } diff --git a/cluster.go b/cluster.go index a5574e2cd..28feb80a3 100644 --- a/cluster.go +++ b/cluster.go @@ -49,14 +49,14 @@ const ( NodeStateLoading = "LOADING" NodeStateReady = "READY" - // ResizeJob states. - ResizeJobStateRunning = "RUNNING" + // resizeJob states. + resizeJobStateRunning = "RUNNING" // Final states. - ResizeJobStateDone = "DONE" - ResizeJobStateAborted = "ABORTED" + resizeJobStateDone = "DONE" + resizeJobStateAborted = "ABORTED" - ResizeJobActionAdd = "ADD" - ResizeJobActionRemove = "REMOVE" + resizeJobActionAdd = "ADD" + resizeJobActionRemove = "REMOVE" ) // Node represents a node in the cluster. @@ -255,8 +255,8 @@ type Cluster struct { joined bool mu sync.RWMutex - jobs map[int64]*ResizeJob - currentJob *ResizeJob + jobs map[int64]*resizeJob + currentJob *resizeJob // Close management wg sync.WaitGroup @@ -279,7 +279,7 @@ func NewCluster() *Cluster { EventReceiver: NopEventReceiver, joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel - jobs: make(map[int64]*ResizeJob), + jobs: make(map[int64]*resizeJob), closing: make(chan struct{}), joining: make(chan struct{}), @@ -289,27 +289,27 @@ func NewCluster() *Cluster { } } -// Coordinator returns the coordinator node. -func (c *Cluster) CoordinatorNode() *Node { - return c.nodeByID(c.Coordinator) +// coordinatorNode returns the coordinator node. +func (c *Cluster) coordinatorNode() *Node { + return c.unprotectedNodeByID(c.Coordinator) } -// IsCoordinator is true if this node is the coordinator. -func (c *Cluster) IsCoordinator() bool { +// isCoordinator is true if this node is the coordinator. +func (c *Cluster) isCoordinator() bool { c.mu.RLock() defer c.mu.RUnlock() - return c.isCoordinator() + return c.unprotectedIsCoordinator() } -func (c *Cluster) isCoordinator() bool { +func (c *Cluster) unprotectedIsCoordinator() bool { return c.Coordinator == c.Node.ID } -// SetCoordinator tells the current node to become the +// setCoordinator tells the current node to become the // Coordinator. In response to this, the current node // will consider itself coordinator and update the other // nodes with its version of Cluster.Status. -func (c *Cluster) SetCoordinator(n *Node) error { +func (c *Cluster) setCoordinator(n *Node) error { c.mu.Lock() // Verify that the new Coordinator value matches // this node. @@ -319,7 +319,7 @@ func (c *Cluster) SetCoordinator(n *Node) error { } // Update IsCoordinator on all nodes (locally). - _ = c.updateCoordinator(n) + _ = c.unprotectedUpdateCoordinator(n) c.mu.Unlock() // Send the update coordinator message to all nodes. err := c.Broadcaster.SendSync( @@ -334,17 +334,17 @@ func (c *Cluster) SetCoordinator(n *Node) error { return c.Broadcaster.SendSync(c.Status()) } -// UpdateCoordinator updates this nodes Coordinator value as well as +// updateCoordinator updates this nodes Coordinator value as well as // changing the corresponding node's IsCoordinator value // to true, and sets all other nodes to false. Returns true if the value // changed. -func (c *Cluster) UpdateCoordinator(n *Node) bool { +func (c *Cluster) updateCoordinator(n *Node) bool { c.mu.Lock() defer c.mu.Unlock() - return c.updateCoordinator(n) + return c.unprotectedUpdateCoordinator(n) } -func (c *Cluster) updateCoordinator(n *Node) bool { +func (c *Cluster) unprotectedUpdateCoordinator(n *Node) bool { var changed bool if c.Coordinator != n.ID { c.Coordinator = n.ID @@ -360,9 +360,9 @@ func (c *Cluster) updateCoordinator(n *Node) bool { return changed } -// AddNode adds a node to the Cluster and updates and saves the +// addNode adds a node to the Cluster and updates and saves the // new topology. -func (c *Cluster) AddNode(node *Node) error { +func (c *Cluster) addNode(node *Node) error { c.Logger.Printf("add node %s to cluster on %s", node, c.Node) // If the node being added is the coordinator, set it for this node. @@ -387,9 +387,9 @@ func (c *Cluster) AddNode(node *Node) error { return c.saveTopology() } -// RemoveNode removes a node from the Cluster and updates and saves the +// removeNode removes a node from the Cluster and updates and saves the // new topology. -func (c *Cluster) RemoveNode(node *Node) error { +func (c *Cluster) removeNode(node *Node) error { // remove from cluster if !c.removeNodeBasicSorted(node) { return nil @@ -407,8 +407,8 @@ func (c *Cluster) RemoveNode(node *Node) error { return c.saveTopology() } -// NodeIDs returns the list of IDs in the cluster. -func (c *Cluster) NodeIDs() []string { +// nodeIDs returns the list of IDs in the cluster. +func (c *Cluster) nodeIDs() []string { return Nodes(c.Nodes).IDs() } @@ -472,9 +472,9 @@ func (c *Cluster) setState(state string) { } } -func (c *Cluster) SetNodeState(state string) error { - if c.IsCoordinator() { - return c.ReceiveNodeState(c.Node.ID, state) +func (c *Cluster) setNodeState(state string) error { + if c.isCoordinator() { + return c.receiveNodeState(c.Node.ID, state) } // Send node state to coordinator. @@ -484,18 +484,18 @@ func (c *Cluster) SetNodeState(state string) error { } c.Logger.Printf("Sending State %s (%s)", state, c.Coordinator) - if err := c.sendTo(c.CoordinatorNode(), ns); err != nil { + if err := c.sendTo(c.coordinatorNode(), ns); err != nil { return fmt.Errorf("sending node state error: err=%s", err) } return nil } -// ReceiveNodeState sets node state in Topology in order for the +// receiveNodeState sets node state in Topology in order for the // Coordinator to keep track of, during startup, which nodes have // finished opening their Holder. -func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { - if !c.IsCoordinator() { +func (c *Cluster) receiveNodeState(nodeID string, state string) error { + if !c.isCoordinator() { return nil } @@ -515,11 +515,6 @@ func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { return nil } -// localNode is not being used. -//func (c *Cluster) localNode() *Node { -// return c.NodeByURI(c.URI) -//} - // Status returns the internal ClusterStatus representation. func (c *Cluster) Status() *internal.ClusterStatus { return &internal.ClusterStatus{ @@ -529,14 +524,14 @@ func (c *Cluster) Status() *internal.ClusterStatus { } } -func (c *Cluster) NodeByID(id string) *Node { +func (c *Cluster) nodeByID(id string) *Node { c.mu.RLock() defer c.mu.RUnlock() - return c.nodeByID(id) + return c.unprotectedNodeByID(id) } -// nodeByID returns a node reference by ID. -func (c *Cluster) nodeByID(id string) *Node { +// unprotectedNodeByID returns a node reference by ID. +func (c *Cluster) unprotectedNodeByID(id string) *Node { for _, n := range c.Nodes { if n.ID == id { return n @@ -558,7 +553,7 @@ func (c *Cluster) nodePositionByID(nodeID string) int { // addNodeBasicSorted adds a node to the cluster, sorted by id. // Returns a pointer to the node and true if the node was added. func (c *Cluster) addNodeBasicSorted(node *Node) bool { - n := c.nodeByID(node.ID) + n := c.unprotectedNodeByID(node.ID) if n != nil { return false } @@ -645,7 +640,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost { func (c *Cluster) fragCombos(idx string, maxSlice uint64, fieldViews viewsByField) fragsByHost { t := make(fragsByHost) for i := uint64(0); i <= maxSlice; i++ { - nodes := c.SliceNodes(idx, i) + nodes := c.sliceNodes(idx, i) for _, n := range nodes { // for each field/view combination: for field, views := range fieldViews { @@ -673,10 +668,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) if lenTo-lenFrom > 1 { return "", "", errors.New("adding more than one node at a time is not supported") } - action = ResizeJobActionAdd + action = resizeJobActionAdd // Determine the node ID that is being added. for _, n := range other.Nodes { - if c.nodeByID(n.ID) == nil { + if c.unprotectedNodeByID(n.ID) == nil { nodeID = n.ID break } @@ -686,10 +681,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) if lenFrom-lenTo > 1 { return "", "", errors.New("removing more than one node at a time is not supported") } - action = ResizeJobActionRemove + action = resizeJobActionRemove // Determine the node ID that is being removed. for _, n := range c.Nodes { - if other.nodeByID(n.ID) == nil { + if other.unprotectedNodeByID(n.ID) == nil { nodeID = n.ID break } @@ -721,7 +716,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R // If a node is being removed, however, then it will most likely // require that a replica fragment be the source data. srcCluster := c - if action == ResizeJobActionAdd && c.ReplicaN > 1 { + if action == resizeJobActionAdd && c.ReplicaN > 1 { srcCluster = NewCluster() srcCluster.Nodes = Nodes(c.Nodes).Clone() srcCluster.Hasher = c.Hasher @@ -740,7 +735,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R srcNodesByFrag := make(map[frag]string) for nodeID, frags := range srcFrags { // If a node is being removed, don't consider it as a source. - if action == ResizeJobActionRemove && nodeID == diffNodeID { + if action == resizeJobActionRemove && nodeID == diffNodeID { continue } for _, frag := range frags { @@ -772,7 +767,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R } src := &internal.ResizeSource{ - Node: EncodeNode(c.nodeByID(srcNodeID)), + Node: EncodeNode(c.unprotectedNodeByID(srcNodeID)), Index: idx.Name(), Field: frag.field, View: frag.view, @@ -786,8 +781,8 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R return m, nil } -// Partition returns the partition that a slice belongs to. -func (c *Cluster) Partition(index string, slice uint64) int { +// partition returns the partition that a slice belongs to. +func (c *Cluster) partition(index string, slice uint64) int { var buf [8]byte binary.BigEndian.PutUint64(buf[:], slice) @@ -798,18 +793,18 @@ func (c *Cluster) Partition(index string, slice uint64) int { return int(h.Sum64() % uint64(c.PartitionN)) } -// SliceNodes returns a list of nodes that own a fragment. -func (c *Cluster) SliceNodes(index string, slice uint64) []*Node { - return c.PartitionNodes(c.Partition(index, slice)) +// sliceNodes returns a list of nodes that own a fragment. +func (c *Cluster) sliceNodes(index string, slice uint64) []*Node { + return c.partitionNodes(c.partition(index, slice)) } -// OwnsSlice returns true if a host owns a fragment. -func (c *Cluster) OwnsSlice(nodeID string, index string, slice uint64) bool { - return Nodes(c.SliceNodes(index, slice)).ContainsID(nodeID) +// ownsSlice returns true if a host owns a fragment. +func (c *Cluster) ownsSlice(nodeID string, index string, slice uint64) bool { + return Nodes(c.sliceNodes(index, slice)).ContainsID(nodeID) } -// PartitionNodes returns a list of nodes that own a partition. -func (c *Cluster) PartitionNodes(partitionID int) []*Node { +// partitionNodes returns a list of nodes that own a partition. +func (c *Cluster) partitionNodes(partitionID int) []*Node { // Default replica count to between one and the number of nodes. // The replica count can be zero if there are no nodes. replicaN := c.ReplicaN @@ -831,27 +826,13 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } -// OwnsSlices finds the set of slices owned by the node per Index -func (c *Cluster) OwnsSlices(index string, maxSlice uint64, uri URI) []uint64 { +// containsSlices is like OwnsSlices, but it includes replicas. +func (c *Cluster) containsSlices(index string, maxSlice uint64, node *Node) []uint64 { var slices []uint64 for i := uint64(0); i <= maxSlice; i++ { - p := c.Partition(index, i) - // Determine primary owner node. - nodeIndex := c.Hasher.Hash(uint64(p), len(c.Nodes)) - if c.Nodes[nodeIndex].URI == uri { - slices = append(slices, i) - } - } - return slices -} - -// ContainsSlices is like OwnsSlices, but it includes replicas. -func (c *Cluster) ContainsSlices(index string, maxSlice uint64, node *Node) []uint64 { - var slices []uint64 - for i := uint64(0); i <= maxSlice; i++ { - p := c.Partition(index, i) + p := c.partition(index, i) // Determine the nodes for partition. - nodes := c.PartitionNodes(p) + nodes := c.partitionNodes(p) for _, n := range nodes { if n.ID == node.ID { slices = append(slices, i) @@ -884,7 +865,7 @@ func (h *jmphasher) Hash(key uint64, n int) int { return int(b) } -func (c *Cluster) Open() error { +func (c *Cluster) open() error { // Cluster always comes up in state STARTING until cluster membership is determined. c.state = ClusterStateStarting @@ -896,7 +877,7 @@ func (c *Cluster) Open() error { c.ID = c.Topology.ClusterID // Only the coordinator needs to consider the .topology file. - if c.IsCoordinator() { + if c.isCoordinator() { err := c.considerTopology() if err != nil { return fmt.Errorf("considerTopology: %v", err) @@ -904,7 +885,7 @@ func (c *Cluster) Open() error { } // Add the local node to the cluster. - err := c.AddNode(c.Node) + err := c.addNode(c.Node) if err != nil { return errors.Wrap(err, "adding local node") } @@ -920,7 +901,7 @@ func (c *Cluster) Open() error { } // If not coordinator then wait for ClusterStatus from coordinator. - if !c.IsCoordinator() { + if !c.isCoordinator() { // In the case where a node has been restarted and memberlist has // not had enough time to determine the node went down/up, then // the coorninator needs to be alerted that this node is back up @@ -945,7 +926,7 @@ func (c *Cluster) Open() error { return nil } -func (c *Cluster) Close() error { +func (c *Cluster) close() error { // Notify goroutines of closing and wait for completion. close(c.closing) c.wg.Wait() @@ -962,14 +943,14 @@ func (c *Cluster) markAsJoined() { } func (c *Cluster) needTopologyAgreement() bool { - return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) + return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) } func (c *Cluster) haveTopologyAgreement() bool { if c.Static { return true } - return StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) + return StringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) } func (c *Cluster) allNodesReady() bool { @@ -999,10 +980,10 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { // channel, which is not consumed until the code below. var eg errgroup.Group eg.Go(func() error { - return j.Run() + return j.run() }) - // Wait for the ResizeJob to finish or be aborted. + // Wait for the resizeJob to finish or be aborted. c.Logger.Printf("wait for jobResult") jobResult := <-j.result @@ -1013,18 +994,18 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { c.Logger.Printf("received jobResult: %s", jobResult) switch jobResult { - case ResizeJobStateDone: - if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil { + case resizeJobStateDone: + if err := c.completeCurrentJob(resizeJobStateDone); err != nil { return errors.Wrap(err, "completing finished job") } // Add/remove uri to/from the cluster. - if j.action == ResizeJobActionRemove { - return c.RemoveNode(nodeAction.node) - } else if j.action == ResizeJobActionAdd { - return c.AddNode(nodeAction.node) + if j.action == resizeJobActionRemove { + return c.removeNode(nodeAction.node) + } else if j.action == resizeJobActionAdd { + return c.addNode(nodeAction.node) } - case ResizeJobStateAborted: - if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil { + case resizeJobStateAborted: + if err := c.completeCurrentJob(resizeJobStateAborted); err != nil { return errors.Wrap(err, "completing aborted job") } } @@ -1045,64 +1026,63 @@ func (c *Cluster) sendTo(node *Node, msg proto.Message) error { return nil } -// ListenForJoins handles cluster-resize events. -func (c *Cluster) ListenForJoins() { - c.wg.Add(1) - go func() { defer c.wg.Done(); c.listenForJoins() }() -} - +// listenForJoins handles cluster-resize events. func (c *Cluster) listenForJoins() { - // When a cluster starts, the state is STARTING. - // We first want to wait for at least one node to join. - // Then we want to clear out the joiningLeavingNodes queue (buffered channel). - // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. - // We use a bool `setNormal` to indicate when at least one node has joined. + c.wg.Add(1) + go func() { + defer c.wg.Done() - var setNormal bool + // When a cluster starts, the state is STARTING. + // We first want to wait for at least one node to join. + // Then we want to clear out the joiningLeavingNodes queue (buffered channel). + // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. + // We use a bool `setNormal` to indicate when at least one node has joined. + var setNormal bool - for { + for { - // Handle all pending joins before changing state back to NORMAL. - select { - case nodeAction := <-c.joiningLeavingNodes: - err := c.handleNodeAction(nodeAction) - if err != nil { - c.Logger.Printf("handleNodeAction error: err=%s", err) + // Handle all pending joins before changing state back to NORMAL. + select { + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) + if err != nil { + c.Logger.Printf("handleNodeAction error: err=%s", err) + continue + } + setNormal = true + continue + default: + } + + // Only change state to NORMAL if we have successfully added at least one host. + if setNormal { + // Put the cluster back to state NORMAL and broadcast. + if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { + c.Logger.Printf("setStateAndBroadcast error: err=%s", err) + } + } + + // Wait for a joining host or a close. + select { + case <-c.closing: + return + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) + if err != nil { + c.Logger.Printf("handleNodeAction error: err=%s", err) + continue + } + setNormal = true continue } - setNormal = true - continue - default: } - - // Only change state to NORMAL if we have successfully added at least one host. - if setNormal { - // Put the cluster back to state NORMAL and broadcast. - if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { - c.Logger.Printf("setStateAndBroadcast error: err=%s", err) - } - } - - // Wait for a joining host or a close. - select { - case <-c.closing: - return - case nodeAction := <-c.joiningLeavingNodes: - err := c.handleNodeAction(nodeAction) - if err != nil { - c.Logger.Printf("handleNodeAction error: err=%s", err) - continue - } - setNormal = true - continue - } - } + }() } -// generateResizeJob creates a new ResizeJob based on the new node being -// added/removed. It also saves a reference to the ResizeJob in the `jobs` map +// generateResizeJob creates a new resizeJob based on the new node being +// added/removed. It also saves a reference to the resizeJob in the `jobs` map // for future lookup by JobID. -func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { +func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*resizeJob, error) { c.Logger.Printf("generateResizeJob: %v", nodeAction) c.mu.Lock() defer c.mu.Unlock() @@ -1111,7 +1091,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { if err != nil { return nil, errors.Wrap(err, "generating job") } - c.Logger.Printf("generated ResizeJob: %d", j.ID) + c.Logger.Printf("generated resizeJob: %d", j.ID) // Save job in jobs map for future reference. c.jobs[j.ID] = j @@ -1125,12 +1105,12 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { return j, nil } -// generateResizeJobByAction returns a ResizeJob with instructions based on +// generateResizeJobByAction returns a resizeJob with instructions based on // the difference between Cluster and a new Cluster with/without uri. -// Broadcaster is associated to the ResizeJob here for use in broadcasting +// Broadcaster is associated to the resizeJob here for use in broadcasting // the resize instructions to other nodes in the cluster. -func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, error) { - j := NewResizeJob(c.Nodes, nodeAction.node, nodeAction.action) +func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, error) { + j := newResizeJob(c.Nodes, nodeAction.node, nodeAction.action) j.Broadcaster = c.Broadcaster // toCluster is a clone of Cluster with the new node added/removed for comparison. @@ -1139,9 +1119,9 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, toCluster.Hasher = c.Hasher toCluster.PartitionN = c.PartitionN toCluster.ReplicaN = c.ReplicaN - if nodeAction.action == ResizeJobActionRemove { + if nodeAction.action == resizeJobActionRemove { toCluster.removeNodeBasicSorted(nodeAction.node) - } else if nodeAction.action == ResizeJobActionAdd { + } else if nodeAction.action == resizeJobActionAdd { toCluster.addNodeBasicSorted(nodeAction.node) } @@ -1172,8 +1152,8 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, } instr := &internal.ResizeInstruction{ JobID: j.ID, - Node: EncodeNode(toCluster.nodeByID(id)), - Coordinator: EncodeNode(c.CoordinatorNode()), + Node: EncodeNode(toCluster.unprotectedNodeByID(id)), + Coordinator: EncodeNode(c.coordinatorNode()), Sources: sources, Schema: c.Holder.EncodeSchema(), // Include the schema to ensure it's in sync on the receiving node. ClusterStatus: c.Status(), @@ -1184,28 +1164,28 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, return j, nil } -// CompleteCurrentJob sets the state of the current ResizeJob +// completeCurrentJob sets the state of the current resizeJob // then removes the pointer to currentJob. -func (c *Cluster) CompleteCurrentJob(state string) error { +func (c *Cluster) completeCurrentJob(state string) error { c.mu.Lock() defer c.mu.Unlock() - if !c.isCoordinator() { + if !c.unprotectedIsCoordinator() { return ErrNodeNotCoordinator } if c.currentJob == nil { return ErrResizeNotRunning } - c.currentJob.SetState(state) + c.currentJob.setState(state) c.currentJob = nil return nil } -// FollowResizeInstruction is run by any node that receives a ResizeInstruction. -func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { +// followResizeInstruction is run by any node that receives a ResizeInstruction. +func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) error { c.Logger.Printf("follow resize instruction on %s", c.Node.ID) // Make sure the cluster status on this node agrees with the Coordinator // before attempting a resize. - if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil { + if err := c.mergeClusterStatus(instr.ClusterStatus); err != nil { return errors.Wrap(err, "merging cluster status") } @@ -1297,13 +1277,13 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err return nil } -func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { +func (c *Cluster) markResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { - j := c.Job(complete.JobID) + j := c.job(complete.JobID) // Abort the job if an error exists in the complete object. if complete.Error != "" { - j.result <- ResizeJobStateAborted + j.result <- resizeJobStateAborted return errors.New(complete.Error) } @@ -1311,29 +1291,27 @@ func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstruc defer j.mu.Unlock() if j.isComplete() { - return fmt.Errorf("ResizeJob %d is no longer running", j.ID) + return fmt.Errorf("resize job %d is no longer running", j.ID) } // Mark host complete. j.IDs[complete.Node.ID] = true if !j.nodesArePending() { - j.result <- ResizeJobStateDone + j.result <- resizeJobStateDone } return nil } -// Job returns a ResizeJob by id. -func (c *Cluster) Job(id int64) *ResizeJob { +// job returns a resizeJob by id. +func (c *Cluster) job(id int64) *resizeJob { c.mu.RLock() defer c.mu.RUnlock() - return c.job(id) + return c.jobs[id] } -func (c *Cluster) job(id int64) *ResizeJob { return c.jobs[id] } - -type ResizeJob struct { +type resizeJob struct { ID int64 IDs map[string]bool Instructions []*internal.ResizeInstruction @@ -1348,15 +1326,15 @@ type ResizeJob struct { Logger Logger } -// NewResizeJob returns a new instance of ResizeJob. -func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { +// newResizeJob returns a new instance of resizeJob. +func newResizeJob(existingNodes []*Node, node *Node, action string) *resizeJob { // Build a map of uris to track their resize status. // The value for a node will be set to true after that node // has indicated that it has completed all resize instructions. ids := make(map[string]bool) - if action == ResizeJobActionRemove { + if action == resizeJobActionRemove { for _, n := range existingNodes { // Exclude the removed node from the map. if n.ID == node.ID { @@ -1364,7 +1342,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { } ids[n.ID] = false } - } else if action == ResizeJobActionAdd { + } else if action == resizeJobActionAdd { for _, n := range existingNodes { ids[n.ID] = false } @@ -1372,7 +1350,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { ids[node.ID] = false } - return &ResizeJob{ + return &resizeJob{ ID: rand.Int63(), IDs: ids, action: action, @@ -1381,50 +1359,40 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { } } -func (j *ResizeJob) State() string { - j.mu.RLock() - defer j.mu.RUnlock() - return j.state -} - -func (j *ResizeJob) SetState(state string) { +func (j *resizeJob) setState(state string) { j.mu.Lock() - j.setState(state) + if j.state == "" || j.state == resizeJobStateRunning { + j.state = state + } j.mu.Unlock() } -func (j *ResizeJob) setState(state string) { - if j.state == "" || j.state == ResizeJobStateRunning { - j.state = state - } -} - -// Run distributes ResizeInstructions. -func (j *ResizeJob) Run() error { - j.Logger.Printf("run ResizeJob") +// run distributes ResizeInstructions. +func (j *resizeJob) run() error { + j.Logger.Printf("run resizeJob") // Set job state to RUNNING. - j.SetState(ResizeJobStateRunning) + j.setState(resizeJobStateRunning) // Job can be considered done in the case where it doesn't require any action. if !j.nodesArePending() { - j.Logger.Printf("ResizeJob contains no pending tasks; mark as done") - j.result <- ResizeJobStateDone + j.Logger.Printf("resizeJob contains no pending tasks; mark as done") + j.result <- resizeJobStateDone return nil } - j.Logger.Printf("distribute tasks for ResizeJob") + j.Logger.Printf("distribute tasks for resizeJob") err := j.distributeResizeInstructions() if err != nil { - j.result <- ResizeJobStateAborted + j.result <- resizeJobStateAborted return errors.Wrap(err, "distributing instructions") } return nil } // isComplete return true if the job is any one of several completion states. -func (j *ResizeJob) isComplete() bool { +func (j *resizeJob) isComplete() bool { switch j.state { - case ResizeJobStateDone, ResizeJobStateAborted: + case resizeJobStateDone, resizeJobStateAborted: return true default: return false @@ -1432,7 +1400,7 @@ func (j *ResizeJob) isComplete() bool { } // nodesArePending returns true if any node is still working on the resize. -func (j *ResizeJob) nodesArePending() bool { +func (j *resizeJob) nodesArePending() bool { for _, complete := range j.IDs { if !complete { return true @@ -1441,9 +1409,9 @@ func (j *ResizeJob) nodesArePending() bool { return false } -func (j *ResizeJob) distributeResizeInstructions() error { +func (j *resizeJob) distributeResizeInstructions() error { j.Logger.Printf("distributeResizeInstructions for job %d", j.ID) - // Loop through the ResizeInstructions in ResizeJob and send to each host. + // Loop through the ResizeInstructions in resizeJob and send to each host. for _, instr := range j.Instructions { // Because the node may not be in the cluster yet, create // a dummy node object to use in the SendTo() method. @@ -1659,7 +1627,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { case NodeJoin: c.Logger.Printf("received NodeJoin event: %v", e) // Ignore the event if this is not the coordinator. - if !c.IsCoordinator() { + if !c.isCoordinator() { return nil } return c.nodeJoin(e.Node) @@ -1681,7 +1649,7 @@ func (c *Cluster) nodeJoin(node *Node) error { return errors.New(err) } - if err := c.AddNode(node); err != nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node for agreement") } @@ -1711,13 +1679,13 @@ func (c *Cluster) nodeJoin(node *Node) error { // If the cluster already contains the node, just send it the cluster status. // This is useful in the case where a node is restarted or temporarily leaves // the cluster. - if node := c.nodeByID(node.ID); node != nil { + if node := c.unprotectedNodeByID(node.ID); node != nil { return c.sendTo(node, c.Status()) } // If the holder does not yet contain data, go ahead and add the node. if ok, err := c.Holder.HasData(); !ok && err == nil { - if err := c.AddNode(node); err != nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") } return c.setStateAndBroadcast(ClusterStateNormal) @@ -1730,16 +1698,16 @@ func (c *Cluster) nodeJoin(node *Node) error { if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { return errors.Wrap(err, "broadcasting state") } - c.joiningLeavingNodes <- nodeAction{node, ResizeJobActionAdd} + c.joiningLeavingNodes <- nodeAction{node, resizeJobActionAdd} return nil } -// NodeLeave initiates the removal of a node from the cluster. -func (c *Cluster) NodeLeave(node *Node) error { +// nodeLeave initiates the removal of a node from the cluster. +func (c *Cluster) nodeLeave(node *Node) error { // Refuse the request if this is not the coordinator. - if !c.IsCoordinator() { - return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.CoordinatorNode().ID) + if !c.isCoordinator() { + return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.coordinatorNode().ID) } if c.State() != ClusterStateNormal { @@ -1747,7 +1715,7 @@ func (c *Cluster) NodeLeave(node *Node) error { } // Ensure that node is in the cluster. - if c.nodeByID(node.ID) == nil { + if c.unprotectedNodeByID(node.ID) == nil { return fmt.Errorf("Node is not a member of the cluster: %s", node.ID) } @@ -1757,18 +1725,12 @@ func (c *Cluster) NodeLeave(node *Node) error { } // See if resize job can be generated - _, err := c.generateResizeJobByAction(nodeAction{c.nodeByID(node.ID), ResizeJobActionRemove}) - - if err != nil { + if _, err := c.generateResizeJobByAction(nodeAction{c.unprotectedNodeByID(node.ID), resizeJobActionRemove}); err != nil { return errors.Wrap(err, "generating job") } - return c.nodeLeave(node) -} - -func (c *Cluster) nodeLeave(node *Node) error { // Get the actual node in the local cluster. - n := c.nodeByID(node.ID) + n := c.unprotectedNodeByID(node.ID) // Don't do anything else if the cluster doesn't contain the node. if n == nil { @@ -1777,7 +1739,7 @@ func (c *Cluster) nodeLeave(node *Node) error { // If the holder does not yet contain data, go ahead and remove the node. if ok, err := c.Holder.HasData(); !ok && err == nil { - if err := c.RemoveNode(n); err != nil { + if err := c.removeNode(n); err != nil { return errors.Wrap(err, "removing node") } return c.setStateAndBroadcast(ClusterStateNormal) @@ -1790,17 +1752,17 @@ func (c *Cluster) nodeLeave(node *Node) error { if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { return errors.Wrap(err, "broadcasting state") } - c.joiningLeavingNodes <- nodeAction{n, ResizeJobActionRemove} + c.joiningLeavingNodes <- nodeAction{n, resizeJobActionRemove} return nil } -func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { +func (c *Cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { c.mu.Lock() defer c.mu.Unlock() c.Logger.Printf("merge cluster status: %v", cs) // Ignore status updates from self (coordinator). - if c.isCoordinator() { + if c.unprotectedIsCoordinator() { return nil } @@ -1811,7 +1773,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { // Add all nodes from the coordinator. for _, node := range officialNodes { - if err := c.AddNode(node); err != nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") } } @@ -1832,7 +1794,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { } for _, nodeID := range nodeIDsToRemove { - if err := c.RemoveNode(c.nodeByID(nodeID)); err != nil { + if err := c.removeNode(c.unprotectedNodeByID(nodeID)); err != nil { return errors.Wrap(err, "removing node") } } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index f97775b96..1fd238911 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -15,11 +15,15 @@ package pilosa import ( + "bytes" "io/ioutil" + "math/rand" "reflect" "strings" "testing" + "testing/quick" + "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/internal" ) @@ -287,19 +291,19 @@ func TestResizeJob(t *testing.T) { { existingNodes: []*Node{node0, node1}, node: node2, - action: ResizeJobActionAdd, + action: resizeJobActionAdd, expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false}, }, { existingNodes: []*Node{node0, node1, node2}, node: node2, - action: ResizeJobActionRemove, + action: resizeJobActionRemove, expectedIDs: map[string]bool{node0.ID: false, node1.ID: false}, }, } for _, test := range tests { - actual := NewResizeJob(test.existingNodes, test.node, test.action) + actual := newResizeJob(test.existingNodes, test.node, test.action) if err != nil { t.Fatal(err) } @@ -308,3 +312,463 @@ func TestResizeJob(t *testing.T) { } } } + +// Ensure the cluster can fairly distribute partitions across the nodes. +func TestCluster_Owners(t *testing.T) { + c := Cluster{ + Nodes: []*Node{ + {URI: NewTestURIFromHostPort("serverA", 1000)}, + {URI: NewTestURIFromHostPort("serverB", 1000)}, + {URI: NewTestURIFromHostPort("serverC", 1000)}, + }, + Hasher: NewTestModHasher(), + ReplicaN: 2, + } + + // Verify nodes are distributed. + if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) { + t.Fatalf("unexpected owners: %s", spew.Sdump(a)) + } + + // Verify nodes go around the ring. + if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) { + t.Fatalf("unexpected owners: %s", spew.Sdump(a)) + } +} + +// Ensure the partitioner can assign a fragment to a partition. +func TestCluster_Partition(t *testing.T) { + if err := quick.Check(func(index string, slice uint64, partitionN int) bool { + c := NewCluster() + c.PartitionN = partitionN + + partitionID := c.partition(index, slice) + if partitionID < 0 || partitionID >= partitionN { + t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN) + } + + return true + }, &quick.Config{ + Values: func(values []reflect.Value, rand *rand.Rand) { + values[0], _ = quick.Value(reflect.TypeOf(""), rand) + values[1] = reflect.ValueOf(uint64(rand.Uint32())) + values[2] = reflect.ValueOf(rand.Intn(1000) + 1) + }, + }); err != nil { + t.Fatal(err) + } +} + +// Ensure the hasher can hash correctly. +func TestHasher(t *testing.T) { + for _, tt := range []struct { + key uint64 + bucket []int + }{ + // Generated from the reference C++ code + {0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}}, + {0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}}, + {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, + } { + for i, v := range tt.bucket { + if got := NewHasher().Hash(tt.key, i+1); got != v { + t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) + } + } + } +} + +// Ensure ContainsSlices can find the actual slice list for node and index. +func TestCluster_ContainsSlices(t *testing.T) { + c := NewTestCluster(5) + c.ReplicaN = 3 + slices := c.containsSlices("test", 10, c.Nodes[2]) + + if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { + t.Fatalf("unexpected slices for node's index: %v", slices) + } +} + +func TestCluster_Nodes(t *testing.T) { + uri0 := NewTestURIFromHostPort("node0", 0) + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + uri3 := NewTestURIFromHostPort("node3", 0) + + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + node3 := &Node{ID: "node3", URI: uri3} + + nodes := []*Node{node0, node1, node2} + + t.Run("NodeIDs", func(t *testing.T) { + actual := Nodes(nodes).IDs() + expected := []string{node0.ID, node1.ID, node2.ID} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Filter", func(t *testing.T) { + actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() + expected := []URI{uri0, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("FilterURI", func(t *testing.T) { + actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() + expected := []URI{uri0, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Contains", func(t *testing.T) { + actualTrue := Nodes(nodes).Contains(node1) + actualFalse := Nodes(nodes).Contains(node3) + if !reflect.DeepEqual(actualTrue, true) { + t.Errorf("expected: %v, but got: %v", true, actualTrue) + } + if !reflect.DeepEqual(actualFalse, false) { + t.Errorf("expected: %v, but got: %v", false, actualTrue) + } + }) + + t.Run("Clone", func(t *testing.T) { + clone := Nodes(nodes).Clone() + actual := Nodes(clone).URIs() + expected := []URI{uri0, uri1, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) +} + +// NEXT: move this test to internal and unexport IsCoordinator +func TestCluster_Coordinator(t *testing.T) { + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + + c1 := *NewCluster() + c1.Node = node1 + c1.Coordinator = node1.ID + c2 := *NewCluster() + c2.Node = node2 + c2.Coordinator = node1.ID + + t.Run("IsCoordinator", func(t *testing.T) { + if !c1.isCoordinator() { + t.Errorf("!IsCoordinator error: %v", c1.Node) + } else if c2.isCoordinator() { + t.Errorf("IsCoordinator error: %v", c2.Node) + } + }) +} + +func TestCluster_Topology(t *testing.T) { + c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} + + uri0 := NewTestURIFromHostPort("host0", 0) + uri1 := NewTestURIFromHostPort("host1", 0) + uri2 := NewTestURIFromHostPort("host2", 0) + invalid := NewTestURIFromHostPort("invalid", 0) + + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} + + t.Run("AddNode", func(t *testing.T) { + err := c1.addNode(node1) + if err != nil { + t.Fatal(err) + } + // add the same host. + err = c1.addNode(node1) + if err != nil { + t.Fatal(err) + } + err = c1.addNode(node2) + if err != nil { + t.Fatal(err) + } + + actual := c1.nodeIDs() + expected := []string{node0.ID, node1.ID, node2.ID} + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("ContainsID", func(t *testing.T) { + if !c1.Topology.ContainsID(node1.ID) { + t.Errorf("!ContainsHost error: %v", node1.ID) + } else if c1.Topology.ContainsID(nodeinvalid.ID) { + t.Errorf("ContainsHost error: %v", nodeinvalid.ID) + } + }) +} + +// Ensure that general cluster functionality works as expected. +func TestCluster_ResizeStates(t *testing.T) { + + t.Run("Single node, no data", func(t *testing.T) { + tc := NewClusterCluster(1) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + node := tc.Clusters[0] + + // Ensure that node comes up in state NORMAL. + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + node := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{node.Node.ID}, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node comes up in state NORMAL. + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, not in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + node := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{"some-other-host"}, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]" + err := tc.Open() + if err == nil || err.Error() != expected { + t.Errorf("did not receive expected error: %s", expected) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, no data", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + tc.AddNode(false) + + node0 := tc.Clusters[0] + node1 := tc.Clusters[1] + + // Ensure that nodes comes up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node0.Node.ID, node1.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) + } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + node0 := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{"node0", "node2"}, + } + tc.WriteTopology(node0.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node is in state STARTING before the other node joins. + if node0.State() != ClusterStateStarting { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State()) + } + + // Expect an error by adding a node not in the topology. + expectedError := "host is not in topology: node1" + err := tc.AddNode(false) + if err == nil || err.Error() != expectedError { + t.Errorf("did not receive expected error: %s", expectedError) + } + + tc.AddNode(false) + node2 := tc.Clusters[2] + + // Ensure that node comes up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node2.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, with data", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + node0 := tc.Clusters[0] + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Add Bit Data to node0. + if err := tc.CreateField("i", "f", FieldOptions{}); err != nil { + t.Fatal(err) + } + tc.SetBit("i", "f", "standard", 1, 101, nil) + tc.SetBit("i", "f", "standard", 1, 1300000, nil) + + // Before starting the resize, get the CheckSum to use for + // comparison later. + node0Field := node0.Holder.Field("i", "f") + node0View := node0Field.View("standard") + node0Fragment := node0View.Fragment(1) + node0Checksum := node0Fragment.Checksum() + + // AddNode needs to block until the resize process has completed. + tc.AddNode(false) + node1 := tc.Clusters[1] + + // Ensure that nodes come up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node0.Node.ID, node1.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) + } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) + } + + // Bits + // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. + node1Field := node1.Holder.Field("i", "f") + node1View := node1Field.View("standard") + node1Fragment := node1View.Fragment(1) + + // Ensure checksums are the same. + if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) { + t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) +} + +// Ensures that coordinator can be changed. +func TestCluster_UpdateCoordinator(t *testing.T) { + t.Run("UpdateCoordinator", func(t *testing.T) { + c := NewTestCluster(2) + + oldNode := c.Nodes[0] + newNode := c.Nodes[1] + + // Update coordinator to the same value. + if c.updateCoordinator(oldNode) { + t.Errorf("did not expect coordinator to change") + } else if c.Coordinator != oldNode.ID { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) + } + + // Update coordinator to a new value. + if !c.updateCoordinator(newNode) { + t.Errorf("expected coordinator to change") + } else if c.Coordinator != newNode.ID { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) + } + }) +} diff --git a/cluster_test.go b/cluster_test.go deleted file mode 100644 index a977b3536..000000000 --- a/cluster_test.go +++ /dev/null @@ -1,494 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "math/rand" - "reflect" - "testing" - "testing/quick" - - "github.com/davecgh/go-spew/spew" -) - -// Ensure the cluster can fairly distribute partitions across the nodes. -func TestCluster_Owners(t *testing.T) { - c := Cluster{ - Nodes: []*Node{ - {URI: NewTestURIFromHostPort("serverA", 1000)}, - {URI: NewTestURIFromHostPort("serverB", 1000)}, - {URI: NewTestURIFromHostPort("serverC", 1000)}, - }, - Hasher: NewTestModHasher(), - ReplicaN: 2, - } - - // Verify nodes are distributed. - if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) { - t.Fatalf("unexpected owners: %s", spew.Sdump(a)) - } - - // Verify nodes go around the ring. - if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) { - t.Fatalf("unexpected owners: %s", spew.Sdump(a)) - } -} - -// Ensure the partitioner can assign a fragment to a partition. -func TestCluster_Partition(t *testing.T) { - if err := quick.Check(func(index string, slice uint64, partitionN int) bool { - c := NewCluster() - c.PartitionN = partitionN - - partitionID := c.Partition(index, slice) - if partitionID < 0 || partitionID >= partitionN { - t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN) - } - - return true - }, &quick.Config{ - Values: func(values []reflect.Value, rand *rand.Rand) { - values[0], _ = quick.Value(reflect.TypeOf(""), rand) - values[1] = reflect.ValueOf(uint64(rand.Uint32())) - values[2] = reflect.ValueOf(rand.Intn(1000) + 1) - }, - }); err != nil { - t.Fatal(err) - } -} - -// Ensure the hasher can hash correctly. -func TestHasher(t *testing.T) { - for _, tt := range []struct { - key uint64 - bucket []int - }{ - // Generated from the reference C++ code - {0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, - {1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}}, - {0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}}, - {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, - } { - for i, v := range tt.bucket { - if got := NewHasher().Hash(tt.key, i+1); got != v { - t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) - } - } - } -} - -// Ensure OwnsSlices can find the actual slice list for node and index. -func TestCluster_OwnsSlices(t *testing.T) { - c := NewTestCluster(5) - slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0)) - - if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) { - t.Fatalf("unexpected slices for node's index: %v", slices) - } -} - -// Ensure ContainsSlices can find the actual slice list for node and index. -func TestCluster_ContainsSlices(t *testing.T) { - c := NewTestCluster(5) - c.ReplicaN = 3 - slices := c.ContainsSlices("test", 10, c.Nodes[2]) - - if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { - t.Fatalf("unexpected slices for node's index: %v", slices) - } -} - -func TestCluster_Nodes(t *testing.T) { - uri0 := NewTestURIFromHostPort("node0", 0) - uri1 := NewTestURIFromHostPort("node1", 0) - uri2 := NewTestURIFromHostPort("node2", 0) - uri3 := NewTestURIFromHostPort("node3", 0) - - node0 := &Node{ID: "node0", URI: uri0} - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - node3 := &Node{ID: "node3", URI: uri3} - - nodes := []*Node{node0, node1, node2} - - t.Run("NodeIDs", func(t *testing.T) { - actual := Nodes(nodes).IDs() - expected := []string{node0.ID, node1.ID, node2.ID} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("Filter", func(t *testing.T) { - actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() - expected := []URI{uri0, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("FilterURI", func(t *testing.T) { - actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() - expected := []URI{uri0, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("Contains", func(t *testing.T) { - actualTrue := Nodes(nodes).Contains(node1) - actualFalse := Nodes(nodes).Contains(node3) - if !reflect.DeepEqual(actualTrue, true) { - t.Errorf("expected: %v, but got: %v", true, actualTrue) - } - if !reflect.DeepEqual(actualFalse, false) { - t.Errorf("expected: %v, but got: %v", false, actualTrue) - } - }) - - t.Run("Clone", func(t *testing.T) { - clone := Nodes(nodes).Clone() - actual := Nodes(clone).URIs() - expected := []URI{uri0, uri1, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) -} - -func TestCluster_Coordinator(t *testing.T) { - uri1 := NewTestURIFromHostPort("node1", 0) - uri2 := NewTestURIFromHostPort("node2", 0) - - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - - c1 := *NewCluster() - c1.Node = node1 - c1.Coordinator = node1.ID - c2 := *NewCluster() - c2.Node = node2 - c2.Coordinator = node1.ID - - t.Run("IsCoordinator", func(t *testing.T) { - if !c1.IsCoordinator() { - t.Errorf("!IsCoordinator error: %v", c1.Node) - } else if c2.IsCoordinator() { - t.Errorf("IsCoordinator error: %v", c2.Node) - } - }) -} - -func TestCluster_Topology(t *testing.T) { - c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} - - uri0 := NewTestURIFromHostPort("host0", 0) - uri1 := NewTestURIFromHostPort("host1", 0) - uri2 := NewTestURIFromHostPort("host2", 0) - invalid := NewTestURIFromHostPort("invalid", 0) - - node0 := &Node{ID: "node0", URI: uri0} - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} - - t.Run("AddNode", func(t *testing.T) { - err := c1.AddNode(node1) - if err != nil { - t.Fatal(err) - } - // add the same host. - err = c1.AddNode(node1) - if err != nil { - t.Fatal(err) - } - err = c1.AddNode(node2) - if err != nil { - t.Fatal(err) - } - - actual := c1.NodeIDs() - expected := []string{node0.ID, node1.ID, node2.ID} - - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("ContainsID", func(t *testing.T) { - if !c1.Topology.ContainsID(node1.ID) { - t.Errorf("!ContainsHost error: %v", node1.ID) - } else if c1.Topology.ContainsID(nodeinvalid.ID) { - t.Errorf("ContainsHost error: %v", nodeinvalid.ID) - } - }) -} - -// Ensure that general cluster functionality works as expected. -func TestCluster_ResizeStates(t *testing.T) { - - t.Run("Single node, no data", func(t *testing.T) { - tc := NewClusterCluster(1) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - node := tc.Clusters[0] - - // Ensure that node comes up in state NORMAL. - if node.State() != ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Single node, in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - node := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{node.Node.ID}, - } - tc.WriteTopology(node.Path, top) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Ensure that node comes up in state NORMAL. - if node.State() != ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Single node, not in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - node := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{"some-other-host"}, - } - tc.WriteTopology(node.Path, top) - - // Open TestCluster. - expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]" - err := tc.Open() - if err == nil || err.Error() != expected { - t.Errorf("did not receive expected error: %s", expected) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, no data", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - tc.AddNode(false) - - node0 := tc.Clusters[0] - node1 := tc.Clusters[1] - - // Ensure that nodes comes up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node1.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node0.Node.ID, node1.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) - } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - node0 := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{"node0", "node2"}, - } - tc.WriteTopology(node0.Path, top) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Ensure that node is in state STARTING before the other node joins. - if node0.State() != ClusterStateStarting { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State()) - } - - // Expect an error by adding a node not in the topology. - expectedError := "host is not in topology: node1" - err := tc.AddNode(false) - if err == nil || err.Error() != expectedError { - t.Errorf("did not receive expected error: %s", expectedError) - } - - tc.AddNode(false) - node2 := tc.Clusters[2] - - // Ensure that node comes up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node2.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, with data", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - node0 := tc.Clusters[0] - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Add Bit Data to node0. - if err := tc.CreateField("i", "f", FieldOptions{}); err != nil { - t.Fatal(err) - } - tc.SetBit("i", "f", "standard", 1, 101, nil) - tc.SetBit("i", "f", "standard", 1, 1300000, nil) - - // Before starting the resize, get the CheckSum to use for - // comparison later. - node0Field := node0.Holder.Field("i", "f") - node0View := node0Field.View("standard") - node0Fragment := node0View.Fragment(1) - node0Checksum := node0Fragment.Checksum() - - // AddNode needs to block until the resize process has completed. - tc.AddNode(false) - node1 := tc.Clusters[1] - - // Ensure that nodes come up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node1.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node0.Node.ID, node1.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) - } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) - } - - // Bits - // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. - node1Field := node1.Holder.Field("i", "f") - node1View := node1Field.View("standard") - node1Fragment := node1View.Fragment(1) - - // Ensure checksums are the same. - if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) { - t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) -} - -// Ensures that coordinator can be changed. -func TestCluster_UpdateCoordinator(t *testing.T) { - t.Run("UpdateCoordinator", func(t *testing.T) { - c := NewTestCluster(2) - - oldNode := c.Nodes[0] - newNode := c.Nodes[1] - - // Update coordinator to the same value. - if c.UpdateCoordinator(oldNode) { - t.Errorf("did not expect coordinator to change") - } else if c.Coordinator != oldNode.ID { - t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) - } - - // Update coordinator to a new value. - if !c.UpdateCoordinator(newNode) { - t.Errorf("expected coordinator to change") - } else if c.Coordinator != newNode.ID { - t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) - } - }) -} diff --git a/executor.go b/executor.go index 17fef6165..241b7c8ec 100644 --- a/executor.go +++ b/executor.go @@ -1002,7 +1002,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Field, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, node := range e.Cluster.sliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.ClearBit(view, rowID, colID, nil) @@ -1078,7 +1078,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, node := range e.Cluster.sliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.SetBit(view, rowID, colID, timestamp) @@ -1414,7 +1414,7 @@ func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (m loop: for _, slice := range slices { - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, node := range e.Cluster.sliceNodes(index, slice) { if Nodes(nodes).Contains(node) { m[node] = append(m[node], slice) continue loop @@ -1444,7 +1444,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, if !opt.Remote { nodes = Nodes(e.Cluster.Nodes).Clone() } else { - nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)} + nodes = []*Node{e.Cluster.unprotectedNodeByID(e.Node.ID)} } // Start mapping across all primary owners. diff --git a/executor_test.go b/executor_test.go index 221a554e8..3d0d9854e 100644 --- a/executor_test.go +++ b/executor_test.go @@ -38,7 +38,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ @@ -87,7 +87,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ @@ -113,7 +113,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, 4) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { @@ -127,7 +127,7 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { defer hldr.Close() hldr.SetBit("i", "general", 10, 1) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil { t.Fatalf("Empty Difference query should give error, but got %v", res) } @@ -145,7 +145,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) { @@ -158,7 +158,7 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect()`), nil, nil); err == nil { t.Fatalf("Empty Intersect query should give error, but got %v", res) } @@ -175,7 +175,7 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { @@ -189,7 +189,7 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { defer hldr.Close() hldr.SetBit("i", "general", 10, 0) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { @@ -208,7 +208,7 @@ func TestExecutor_Execute_Xor(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) { @@ -224,7 +224,7 @@ func TestExecutor_Execute_Count(t *testing.T) { hldr.SetBit("i", "f", 10, SliceWidth+1) hldr.SetBit("i", "f", 10, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { @@ -240,7 +240,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { // set a bit so the view gets created. hldr.SetBit("i", "f", 1, 0) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if n := hldr.Row("i", "f", 11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } @@ -284,7 +284,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } // Set bsiGroup values. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f=25)`), nil, nil); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=100, f=10)`), nil, nil); err != nil { @@ -322,21 +322,21 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnBSIGroupValue", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err != pilosa.ErrInvalidBSIGroupValueType { t.Fatalf("unexpected error: %s", err) } @@ -359,7 +359,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two attrs on f/10. // Also set attrs on other bitmaps and fields to test isolation. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } @@ -385,7 +385,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { func TestExecutor_Execute_TopN(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // Set columns for rows 0, 10, & 20 across two slices. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { @@ -437,7 +437,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr.SetBit("i", "f", 1, SliceWidth) // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -471,7 +471,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { hldr.SetBit("i", "f", 4, 3*SliceWidth+1) // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -506,7 +506,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache() // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, field=other), field=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -530,7 +530,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -553,7 +553,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,field=f),field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -567,7 +567,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { func TestExecutor_Execute_MinMax(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -662,7 +662,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { func TestExecutor_Execute_Sum(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -733,7 +733,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { func TestExecutor_Execute_BSIGroupRange(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // Create index. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -775,7 +775,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { func TestExecutor_Execute_Range(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -955,7 +955,7 @@ func TestExecutor_Execute_Range(t *testing.T) { // Ensure a remote query can return a row. func TestExecutor_Execute_Remote_Row(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. s := test.NewServer() @@ -1003,7 +1003,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { // Ensure a remote query can return a count. func TestExecutor_Execute_Remote_Count(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. s := test.NewServer() @@ -1038,7 +1038,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) c.ReplicaN = 2 // Create secondary server and update second cluster node. @@ -1090,7 +1090,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) c.ReplicaN = 2 // Create secondary server and update second cluster node. @@ -1144,7 +1144,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Ensure a remote query can return a top-n query. func TestExecutor_Execute_Remote_TopN(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. s := test.NewServer() @@ -1213,7 +1213,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.MaxWritesPerRequest = 3 if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites { t.Fatalf("unexpected error: %s", err) @@ -1229,7 +1229,7 @@ func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) { targetAttrs := map[string]interface{}{ "foo": "bar", } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // SetColumnAttrs call should exclude the field attribute _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=10)"), nil, nil) diff --git a/fragment.go b/fragment.go index 830033c63..012c62503 100644 --- a/fragment.go +++ b/fragment.go @@ -1740,7 +1740,7 @@ func (s *FragmentSyncer) isClosing() bool { // then merges any blocks which have differences. func (s *FragmentSyncer) syncFragment() error { // Determine replica set. - nodes := s.Cluster.SliceNodes(s.Fragment.index, s.Fragment.slice) + nodes := s.Cluster.sliceNodes(s.Fragment.index, s.Fragment.slice) if len(nodes) == 1 { return nil } @@ -1821,7 +1821,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Read pairs from each remote block. var uris []*URI var pairSets []pairSet - for _, node := range s.Cluster.SliceNodes(f.index, f.slice) { + for _, node := range s.Cluster.sliceNodes(f.index, f.slice) { if s.Node.ID == node.ID { continue } diff --git a/holder.go b/holder.go index b3147d897..e7af65643 100644 --- a/holder.go +++ b/holder.go @@ -619,7 +619,7 @@ func (s *HolderSyncer) SyncHolder() error { for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { // Ignore slices that this host doesn't own. - if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) { + if !s.Cluster.ownsSlice(s.Node.ID, di.Name, slice) { continue } @@ -799,7 +799,7 @@ func (c *HolderCleaner) CleanHolder() error { } // Get the fragments that node is responsible for (based on hash(index, node)). - containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node) + containedSlices := c.Cluster.containsSlices(index.Name(), index.MaxSlice(), c.Node) // Get the fragments registered in memory. for _, field := range index.Fields() { diff --git a/holder_test.go b/holder_test.go index b56863bb0..31b50b900 100644 --- a/holder_test.go +++ b/holder_test.go @@ -383,7 +383,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Mock 2-node, fully replicated cluster. cluster.ReplicaN = 2 - cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) + cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0) cluster.Nodes[1].URI = *uri // Create fields on nodes. @@ -456,7 +456,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Ensure holder can clean up orphaned fragments. func TestHolderCleaner_CleanHolder(t *testing.T) { - cluster := test.NewCluster(2) + cluster := pilosa.NewTestCluster(2) // Create a local holder. hldr0 := test.MustOpenHolder() @@ -465,7 +465,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Mock 2-node, fully replicated cluster. cluster.ReplicaN = 2 - cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) + cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0) // Create fields on nodes. for _, hldr := range []*test.Holder{hldr0} { diff --git a/http/client_test.go b/http/client_test.go index 1cc972ff6..851f1140e 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -87,10 +87,17 @@ func TestClient_MultiNode(t *testing.T) { // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. sliceNums := []uint64{1, 2, 6} + + // This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())` + owns := [][]uint64{ + {1, 3, 4, 8, 10, 13, 17, 19}, + {2, 5, 7, 11, 12, 14, 18}, + {0, 6, 9, 15, 16, 20}, + } + for i, num := range sliceNums { - owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI()) ownsNum := false - for _, ownNum := range owns { + for _, ownNum := range owns[i] { if ownNum == num { ownsNum = true break diff --git a/server.go b/server.go index b1f0f3d10..f330b6e6a 100644 --- a/server.go +++ b/server.go @@ -332,7 +332,7 @@ func (s *Server) Open() error { } // Open Cluster management. - if err := s.Cluster.Open(); err != nil { + if err := s.Cluster.open(); err != nil { return fmt.Errorf("opening Cluster: %v", err) } @@ -340,7 +340,7 @@ func (s *Server) Open() error { if err := s.Holder.Open(); err != nil { return fmt.Errorf("opening Holder: %v", err) } - if err := s.Cluster.SetNodeState(NodeStateReady); err != nil { + if err := s.Cluster.setNodeState(NodeStateReady); err != nil { return fmt.Errorf("setting nodeState: %v", err) } @@ -349,7 +349,7 @@ func (s *Server) Open() error { // the cluster without waiting for data to load on the coordinator. Before // this starts, the joins are queued up in the Cluster.joiningLeavingNodes // buffered channel. - s.Cluster.ListenForJoins() + s.Cluster.listenForJoins() // Start background monitoring. s.wg.Add(3) @@ -370,7 +370,7 @@ func (s *Server) Close() error { s.ln.Close() } if s.Cluster != nil { - s.Cluster.Close() + s.Cluster.close() } if s.Holder != nil { s.Holder.Close() @@ -493,26 +493,26 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return err } case *internal.ClusterStatus: - err := s.Cluster.MergeClusterStatus(obj) + err := s.Cluster.mergeClusterStatus(obj) if err != nil { return err } case *internal.ResizeInstruction: - err := s.Cluster.FollowResizeInstruction(obj) + err := s.Cluster.followResizeInstruction(obj) if err != nil { return err } case *internal.ResizeInstructionComplete: - err := s.Cluster.MarkResizeInstructionComplete(obj) + err := s.Cluster.markResizeInstructionComplete(obj) if err != nil { return err } case *internal.SetCoordinatorMessage: - s.Cluster.SetCoordinator(DecodeNode(obj.New)) + s.Cluster.setCoordinator(DecodeNode(obj.New)) case *internal.UpdateCoordinatorMessage: - s.Cluster.UpdateCoordinator(DecodeNode(obj.New)) + s.Cluster.updateCoordinator(DecodeNode(obj.New)) case *internal.NodeStateMessage: - err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State) + err := s.Cluster.receiveNodeState(obj.NodeID, obj.State) if err != nil { return err } @@ -650,7 +650,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) s.diagnostics.Set("Host", s.URI.host) - s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ",")) + s.diagnostics.Set("Cluster", strings.Join(s.Cluster.nodeIDs(), ",")) s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) s.diagnostics.Set("NodeID", s.NodeID) diff --git a/stats_test.go b/stats_test.go index ddf577918..6644786cf 100644 --- a/stats_test.go +++ b/stats_test.go @@ -95,7 +95,7 @@ func TestStatsCount_TopN(t *testing.T) { // Execute query. called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { if name != "TopN" { @@ -124,7 +124,7 @@ func TestStatsCount_Bitmap(t *testing.T) { hldr.SetBit("d", "f", 0, 0) hldr.SetBit("d", "f", 0, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { if name != "Bitmap" { @@ -154,7 +154,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { hldr.SetBit("d", "f", 10, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) field := e.Holder.Field("d", "f") if field == nil { t.Fatal("field not found") @@ -184,7 +184,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { hldr.SetBit("d", "f", 10, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx := e.Holder.Index("d") if idx == nil { t.Fatal("idex not found") diff --git a/test/cluster.go b/test/cluster.go index 6b1efe665..d1ee6af80 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -15,17 +15,10 @@ package test import ( - "bufio" - "bytes" "fmt" "io/ioutil" - "path/filepath" - "sync" - "time" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" ) // NewCluster returns a cluster with n nodes and uses a mod-based hasher. @@ -37,14 +30,14 @@ func NewCluster(n int) *pilosa.Cluster { c := pilosa.NewCluster() c.ReplicaN = 1 - c.Hasher = NewModHasher() + c.Hasher = newModHasher() c.Path = path c.Topology = pilosa.NewTopology() for i := 0; i < n; i++ { c.Nodes = append(c.Nodes, &pilosa.Node{ ID: fmt.Sprintf("node%d", i), - URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)), + URI: newURI("http", fmt.Sprintf("host%d", i), uint16(0)), }) } @@ -55,372 +48,19 @@ func NewCluster(n int) *pilosa.Cluster { return c } -// ModHasher represents a simple, mod-based hashing. -type ModHasher struct{} +// modHasher represents a simple, mod-based hashing. +type modHasher struct{} -// NewModHasher returns a new instance of ModHasher with n buckets. -func NewModHasher() *ModHasher { return &ModHasher{} } +// newModHasher returns a new instance of ModHasher with n buckets. +func newModHasher() *modHasher { return &modHasher{} } -func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n } +func (*modHasher) Hash(key uint64, n int) int { return int(key) % n } -// ConstHasher represents hash that always returns the same index. -type ConstHasher struct { - i int -} - -// NewConstHasher returns a new instance of ConstHasher that always returns i. -func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} } - -func (h *ConstHasher) Hash(key uint64, n int) int { return h.i } - -// NewURI is a test URI creator that intentionally swallows errors. -func NewURI(scheme, host string, port uint16) pilosa.URI { +// newURI is a test URI creator that intentionally swallows errors. +func newURI(scheme, host string, port uint16) pilosa.URI { uri := pilosa.DefaultURI() uri.SetScheme(scheme) uri.SetHost(host) uri.SetPort(port) return *uri } - -func NewURIFromHostPort(host string, port uint16) pilosa.URI { - uri := pilosa.DefaultURI() - uri.SetHost(host) - uri.SetPort(port) - return *uri -} - -// TestCluster represents a cluster of test nodes, each of which -// has a pilosa.Cluster. -type TestCluster struct { - Clusters []*pilosa.Cluster - - common *commonClusterSettings - - mu sync.RWMutex - resizing bool - resizeDone chan struct{} -} - -type commonClusterSettings struct { - Nodes []*pilosa.Node -} - -func (t *TestCluster) CreateIndex(name string) error { - for _, c := range t.Clusters { - if _, err := c.Holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}); err != nil { - return err - } - } - return nil -} - -func (t *TestCluster) CreateField(index, field string, opt pilosa.FieldOptions) error { - for _, c := range t.Clusters { - idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{}) - if err != nil { - return err - } - if _, err := idx.CreateField(field, opt); err != nil { - return err - } - } - return nil -} -func (t *TestCluster) SetBit(index, field, view string, rowID, colID uint64, x *time.Time) error { - // Determine which node should receive the SetBit. - c0 := t.Clusters[0] // use the first node's cluster to determine slice location. - slice := colID / pilosa.SliceWidth - nodes := c0.SliceNodes(index, slice) - - for _, node := range nodes { - c := t.clusterByID(node.ID) - if c == nil { - continue - } - f := c.Holder.Field(index, field) - if f == nil { - return fmt.Errorf("index/field does not exist: %s/%s", index, field) - } - _, err := f.SetBit(view, rowID, colID, x) - if err != nil { - return err - } - } - - return nil -} - -func (t *TestCluster) clusterByID(id string) *pilosa.Cluster { - for _, c := range t.Clusters { - if c.Node.ID == id { - return c - } - } - return nil -} - -// AddNode adds a node to the cluster and (potentially) starts a resize job. -func (t *TestCluster) AddNode(saveTopology bool) error { - id := len(t.Clusters) - - c, err := t.addCluster(id, saveTopology) - if err != nil { - return err - } - - // Send NodeJoin event to coordinator. - if id > 0 { - coord := t.Clusters[0] - ev := &pilosa.NodeEvent{ - Event: pilosa.NodeJoin, - Node: c.Node, - } - - if err := coord.ReceiveEvent(ev); err != nil { - return err - } - - // Wait for the AddNode job to finish. - if c.State() != pilosa.ClusterStateNormal { - t.resizeDone = make(chan struct{}) - t.mu.Lock() - t.resizing = true - t.mu.Unlock() - <-t.resizeDone - } - } - - return nil -} - -// WriteTopology writes the given topology to disk. -func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error { - if buf, err := proto.Marshal(top.Encode()); err != nil { - return err - } else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil { - return err - } - return nil -} - -func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) { - - id := fmt.Sprintf("node%d", i) - uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0)) - - node := &pilosa.Node{ - ID: id, - URI: uri, - } - - // add URI to common - //t.common.NodeIDs = append(t.common.NodeIDs, id) - //sort.Sort(t.common.NodeIDs) - - // add node to common - t.common.Nodes = append(t.common.Nodes, node) - - // create node-specific temp directory - path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) - if err != nil { - return nil, err - } - - // holder - h := pilosa.NewHolder() - h.Path = path - - // cluster - c := pilosa.NewCluster() - c.ReplicaN = 1 - c.Hasher = NewModHasher() - c.Path = path - c.Topology = pilosa.NewTopology() - c.Holder = h - c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes) - c.Node = node - c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator - c.Broadcaster = t - - // add nodes - if saveTopology { - for _, n := range t.common.Nodes { - c.AddNode(n) - } - } - - // Add this node to the TestCluster. - t.Clusters = append(t.Clusters, c) - - return c, nil -} - -// NewTestCluster returns a new instance of test.Cluster. -func NewTestCluster(n int) *TestCluster { - - tc := &TestCluster{ - common: &commonClusterSettings{}, - } - - // add clusters - for i := 0; i < n; i++ { - _, err := tc.addCluster(i, true) - if err != nil { - panic(err) - } - } - return tc -} - -// SetState sets the state of the cluster on each node. -func (t *TestCluster) SetState(state string) { - for _, c := range t.Clusters { - c.SetState(state) - } -} - -// Open opens all clusters in the test cluster. -func (t *TestCluster) Open() error { - for _, c := range t.Clusters { - if err := c.Open(); err != nil { - return err - } - if err := c.Holder.Open(); err != nil { - return err - } - if err := c.SetNodeState(pilosa.NodeStateReady); err != nil { - return err - } - } - - // Start the listener on the coordinator. - if len(t.Clusters) == 0 { - return nil - } - t.Clusters[0].ListenForJoins() - - return nil -} - -// Close closes all clusters in the test cluster. -func (t *TestCluster) Close() error { - for _, c := range t.Clusters { - err := c.Close() - if err != nil { - return err - } - } - return nil -} - -// TestCluster implements Broadcaster interface. - -// SendSync is a test implemenetation of Broadcaster SendSync method. -func (t *TestCluster) SendSync(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ClusterStatus: - // Apply the send message to all nodes (except the coordinator). - for _, c := range t.Clusters { - c.MergeClusterStatus(obj) - } - t.mu.RLock() - if obj.State == pilosa.ClusterStateNormal && t.resizing { - close(t.resizeDone) - } - t.mu.RUnlock() - } - - return nil -} - -// SendAsync is a test implemenetation of Broadcaster SendAsync method. -func (t *TestCluster) SendAsync(pb proto.Message) error { - return nil -} - -// SendTo is a test implemenetation of Broadcaster SendTo method. -func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ResizeInstruction: - err := t.FollowResizeInstruction(obj) - if err != nil { - return err - } - case *internal.ResizeInstructionComplete: - coord := t.clusterByID(to.ID) - go coord.MarkResizeInstructionComplete(obj) - } - return nil -} - -// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. -func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { - - // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ - JobID: instr.JobID, - Node: instr.Node, - Error: "", - } - - // Stop processing on any error. - if err := func() error { - - // figure out which node it was meant for, then call the operation on that cluster - // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI) - instrNode := pilosa.DecodeNode(instr.Node) - destCluster := t.clusterByID(instrNode.ID) - - // Sync the schema received in the resize instruction. - if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { - return err - } - - for _, src := range instr.Sources { - srcNode := pilosa.DecodeNode(src.Node) - srcCluster := t.clusterByID(srcNode.ID) - - srcFragment := srcCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice) - destFragment := destCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice) - if destFragment == nil { - // Create fragment on destination if it doesn't exist. - f := destCluster.Holder.Field(src.Index, src.Field) - v := f.View(src.View) - var err error - destFragment, err = v.CreateFragmentIfNotExists(src.Slice) - if err != nil { - return err - } - } - - buf := bytes.NewBuffer(nil) - - bw := bufio.NewWriter(buf) - br := bufio.NewReader(buf) - - // Get the fragment from source. - if _, err := srcFragment.WriteTo(bw); err != nil { - return err - } - - // Flush the bufio.buf to the io.Writer (buf). - bw.Flush() - - // Write data to destination. - if _, err := destFragment.ReadFrom(br); err != nil { - return err - } - } - - return nil - }(); err != nil { - complete.Error = err.Error() - } - - node := pilosa.DecodeNode(instr.Coordinator) - if err := t.SendTo(node, complete); err != nil { - return err - } - - return nil -} diff --git a/utils_test.go b/utils_internal_test.go similarity index 97% rename from utils_test.go rename to utils_internal_test.go index 6335d087f..4055ef566 100644 --- a/utils_test.go +++ b/utils_internal_test.go @@ -121,7 +121,7 @@ func (t *ClusterCluster) SetBit(index, field, view string, rowID, colID uint64, // Determine which node should receive the SetBit. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. slice := colID / SliceWidth - nodes := c0.SliceNodes(index, slice) + nodes := c0.sliceNodes(index, slice) for _, node := range nodes { c := t.clusterByID(node.ID) @@ -236,7 +236,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*Cluster, error) // add nodes if saveTopology { for _, n := range t.common.Nodes { - c.AddNode(n) + c.addNode(n) } } @@ -273,13 +273,13 @@ func (t *ClusterCluster) SetState(state string) { // Open opens all clusters in the test cluster. func (t *ClusterCluster) Open() error { for _, c := range t.Clusters { - if err := c.Open(); err != nil { + if err := c.open(); err != nil { return err } if err := c.Holder.Open(); err != nil { return err } - if err := c.SetNodeState(NodeStateReady); err != nil { + if err := c.setNodeState(NodeStateReady); err != nil { return err } } @@ -288,7 +288,7 @@ func (t *ClusterCluster) Open() error { if len(t.Clusters) == 0 { return nil } - t.Clusters[0].ListenForJoins() + t.Clusters[0].listenForJoins() return nil } @@ -296,7 +296,7 @@ func (t *ClusterCluster) Open() error { // Close closes all clusters in the test cluster. func (t *ClusterCluster) Close() error { for _, c := range t.Clusters { - err := c.Close() + err := c.close() if err != nil { return err } @@ -310,7 +310,7 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error { case *internal.ClusterStatus: // Apply the send message to all nodes (except the coordinator). for _, c := range t.Clusters { - c.MergeClusterStatus(obj) + c.mergeClusterStatus(obj) } t.mu.RLock() if obj.State == ClusterStateNormal && t.resizing { @@ -337,7 +337,7 @@ func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { } case *internal.ResizeInstructionComplete: coord := t.clusterByID(to.ID) - go coord.MarkResizeInstructionComplete(obj) + go coord.markResizeInstructionComplete(obj) } return nil } From 1211663019b3cf4fd4497094cecbaca3046fcae1 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 13 Jun 2018 15:47:00 -0500 Subject: [PATCH 66/78] add supporting functions for functional query contstruction --- test/querygenerator_test.go | 403 ++++++++++++++++++++++++++++++++++++ 1 file changed, 403 insertions(+) create mode 100644 test/querygenerator_test.go diff --git a/test/querygenerator_test.go b/test/querygenerator_test.go new file mode 100644 index 000000000..4702e6b91 --- /dev/null +++ b/test/querygenerator_test.go @@ -0,0 +1,403 @@ +package test + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "github.com/pilosa/pilosa/pql" +) + +type Args map[string]interface{} + +type Calls []*pql.Call + +func PQL(calls ...*pql.Call) *pql.Query { + return &pql.Query{Calls: calls} +} + +func Row(frame string, row int) *pql.Call { + return &pql.Call{ + Name: "Row", + Args: Args{ + "frame": frame, + "row": row, + }, + } +} + +func mutationArgs(args ...interface{}) Args { + rargs := make(Args) + for _, arg := range args { + switch v := arg.(type) { + case int: + rargs["column"] = v + case string: + if strings.Contains(v, "=") { + parts := strings.Split(v, "=") + rargs["frame"] = parts[0] + i, _ := strconv.ParseInt(parts[1], 10, 64) + rargs["value"] = i + } else { + rargs["timestamp"] = v + } + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs +} + +func Set(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Set", Args: mutationArgs(args...)} +} + +func Clear(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Clear", Args: mutationArgs(args...)} +} + +func magic(args ...interface{}) (Args, Calls) { + var ( + rargs Args + calls Calls + ) + + for _, arg := range args { + switch v := arg.(type) { + case Args: + rargs = v + case []*pql.Call: + calls = append(calls, v...) + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs, calls +} +func Count(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Count", Args: kvargs, Children: children} +} + +func Union(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Union", Args: kvargs, Children: children} +} + +func Intersect(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Intersect", Args: kvargs, Children: children} +} + +func Difference(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Difference", Args: kvargs, Children: children} +} +func Xor(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Xor", Args: kvargs, Children: children} +} + +func Between(frame string, min, max int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.BETWEEN, + "Value": []int{min, max}, + }, + } +} +func Lt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LT, + "Value": column, + }, + } +} +func Lte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LTE, + "Value": column, + }, + } +} +func Gt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GT, + "Value": column, + }, + } +} + +func Gte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GTE, + "Value": column, + }, + } +} +func CompareCall(a, b *pql.Call) bool { + if a.Name != b.Name { + return false + } + for k, i := range a.Args { + switch v := i.(type) { + case []int: + bside := b.Args[k] + for j := range v { + if v[j] != bside.([]int)[j] { + return false + } + + } + default: + if b.Args[k] != i { + return false + } + } + } + + if len(a.Children) == len(b.Children) { + for i := range a.Children { + if !CompareCall(a.Children[i], b.Children[i]) { + return false + } + } + } else { + return false + } + return true +} + +func Compare(a, b *pql.Query) bool { + for i := range a.Calls { + if !CompareCall(a.Calls[i], b.Calls[i]) { + return false + } + + } + return true +} + +func TestPQL_Generator(t *testing.T) { + t.Run("pql.Query generator", func(t *testing.T) { + for _, u := range []struct { + pql string + calc *pql.Query + exp *pql.Query + }{ + { + pql: "Union(Row(aaa=10),Row(bbb=9))", + calc: PQL(Union(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Union", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Intersect(Row(aaa=10),Row(bbb=9))", + calc: PQL(Intersect(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Intersect", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Difference(Row(aaa=10),Row(bbb=9))", + calc: PQL(Difference(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Difference", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Range(bbb > 20)", + calc: PQL(Gt("bbb", 20)), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Range", + Args: map[string]interface{}{ + "Op": pql.GT, + "Value": 20, + }, + }, + }, + }, + }, + { + pql: "Range(10 < bbb < 20)", + calc: PQL(Between("bbb", 10, 20)), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Range", + Args: map[string]interface{}{ + "Op": pql.BETWEEN, + "Value": []int{10, 20}, + }, + }, + }, + }, + }, + { + pql: "Set(10, aaa=9)", + calc: PQL(Set(10, "aaa=9")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Set", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + }, + }, + }, + }, + }, + { + pql: "Clear(10, aaa=10)", + calc: PQL(Clear(10, "aaa=9")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Clear", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + }, + }, + }, + }, + }, + { + pql: `Set(10, aaa=10, "2017-03-02T03:00")`, + calc: PQL(Set(10, "aaa=9", "2017-03-02T03:00")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Set", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + "timestamp": "2017-03-02T03:00", + }, + }, + }, + }, + }, + { + pql: `Count(Row(aaa=10))`, + calc: PQL(Count(Row("aaa", 10))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Count", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + }, + }, + }, + }, + }, + { + pql: "Intersect(Union(Row(aaa=10),Row(bbb=9)), Row(aaa=12))", + calc: PQL(Intersect(Union(Row("aaa", 10), Row("bbb", 9)), Row("aaa", 12))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Intersect", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Union", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 12}, + }, + }, + }, + }, + }, + }, + } { + + if !Compare(u.calc, u.exp) { + t.Fatalf("Not Equal. expected: %v, got %v for %s", u.exp, u.calc, u.pql) + } + } + }) + +} From fe167ea78c026f32ecf71a2e99748ac199c9fa64 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 11 Jun 2018 13:49:00 -0500 Subject: [PATCH 67/78] un-export some top-level functions --- cluster.go | 4 +- executor.go | 2 +- field.go | 16 +++---- gossip/gossip.go | 21 ++++++++- index.go | 4 +- pilosa.go | 68 ++------------------------- pilosa_internal_test.go | 43 +++++++++++++++++ pilosa_test.go | 48 ------------------- server.go | 12 ++--- server/server_test.go | 16 ------- server_internal_test.go | 35 ++++++++++++++ server_test.go | 14 ++++++ stats.go | 6 +-- statsd/statsd.go | 38 ++++++++++++++- time.go | 28 +++++------ time_test.go => time_internal_test.go | 58 +++++++++++------------ view.go | 4 +- 17 files changed, 219 insertions(+), 198 deletions(-) create mode 100644 pilosa_internal_test.go create mode 100644 server_internal_test.go rename time_test.go => time_internal_test.go (68%) diff --git a/cluster.go b/cluster.go index 28feb80a3..8462d91e7 100644 --- a/cluster.go +++ b/cluster.go @@ -943,14 +943,14 @@ func (c *Cluster) markAsJoined() { } func (c *Cluster) needTopologyAgreement() bool { - return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) + return c.State() == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) } func (c *Cluster) haveTopologyAgreement() bool { if c.Static { return true } - return StringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) + return stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) } func (c *Cluster) allNodesReady() bool { diff --git a/executor.go b/executor.go index 241b7c8ec..03c636215 100644 --- a/executor.go +++ b/executor.go @@ -752,7 +752,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Union bitmaps across all time-based views. row := &Row{} - for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) { + for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) { f := e.Holder.Fragment(index, field, view, slice) if f == nil { continue diff --git a/field.go b/field.go index 8c7683cd1..d03da2c22 100644 --- a/field.go +++ b/field.go @@ -82,7 +82,7 @@ func OptFieldFieldOptions(o FieldOptions) FieldOption { // NewField returns a new instance of field. func NewField(path, index, name string, opts ...FieldOption) (*Field, error) { - err := ValidateName(name) + err := validateName(name) if err != nil { return nil, err } @@ -645,7 +645,7 @@ func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) { // SetBit sets a bit on a view within the field. func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. - if !IsValidView(name) { + if !isValidView(name) { return false, ErrInvalidView } @@ -668,7 +668,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed } // If a timestamp is specified then set bits across all views for the quantum. - for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { + for _, subname := range viewsByTime(name, *t, f.TimeQuantum()) { view, err := f.CreateViewIfNotExists(subname) if err != nil { return changed, errors.Wrapf(err, "creating view %s", subname) @@ -687,7 +687,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed // ClearBit clears a bit within the field. func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. - if !IsValidView(name) { + if !isValidView(name) { return false, ErrInvalidView } @@ -710,7 +710,7 @@ func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (change } // If a timestamp is specified then clear bits across all views for the quantum. - for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { + for _, subname := range viewsByTime(name, *t, f.TimeQuantum()) { view, err := f.CreateViewIfNotExists(subname) if err != nil { return changed, errors.Wrapf(err, "creating view %s", subname) @@ -899,7 +899,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro if timestamp == nil { standard = []string{ViewStandard} } else { - standard = ViewsByTime(ViewStandard, *timestamp, q) + standard = viewsByTime(ViewStandard, *timestamp, q) // In order to match the logic of `SetBit()`, we want bits // with timestamps to write to both time and standard views. standard = append(standard, ViewStandard) @@ -1233,8 +1233,8 @@ const ( CacheTypeNone = "none" ) -// IsValidCacheType returns true if v is a valid cache type. -func IsValidCacheType(v string) bool { +// isValidCacheType returns true if v is a valid cache type. +func isValidCacheType(v string) bool { switch v { case CacheTypeLRU, CacheTypeRanked, CacheTypeNone: return true diff --git a/gossip/gossip.go b/gossip/gossip.go index 2da6e3e4c..dfcb2f759 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -18,6 +18,7 @@ import ( "fmt" "io/ioutil" "log" + "net" "strconv" "strings" "sync" @@ -213,7 +214,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe conf.BindAddr = host conf.BindPort = port conf.AdvertisePort = port - conf.AdvertiseAddr = pilosa.HostToIP(host) + conf.AdvertiseAddr = hostToIP(host) // conf.TCPTimeout = time.Duration(cfg.StreamTimeout) conf.SuspicionMult = cfg.SuspicionMult @@ -580,3 +581,21 @@ type Config struct { Nodes int `toml:"nodes"` ToTheDeadTime toml.Duration `toml:"to-the-dead-time"` } + +// hostToIP converts host to an IP4 address based on net.LookupIP(). +func hostToIP(host string) string { + // if host is not an IP addr, check net.LookupIP() + if net.ParseIP(host) == nil { + hosts, err := net.LookupIP(host) + if err != nil { + return host + } + for _, h := range hosts { + // this restricts pilosa to IP4 + if h.To4() != nil { + return h.String() + } + } + } + return host +} diff --git a/index.go b/index.go index 80a5c2eb8..46ca48e47 100644 --- a/index.go +++ b/index.go @@ -53,7 +53,7 @@ type Index struct { // NewIndex returns a new instance of Index. func NewIndex(path, name string) (*Index, error) { - err := ValidateName(name) + err := validateName(name) if err != nil { return nil, errors.Wrap(err, "validating name") } @@ -295,7 +295,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, e func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { if name == "" { return nil, errors.New("field name required") - } else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) { + } else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) { return nil, ErrInvalidCacheType } diff --git a/pilosa.go b/pilosa.go index a87bab4fc..c18fb5c0c 100644 --- a/pilosa.go +++ b/pilosa.go @@ -16,9 +16,7 @@ package pilosa import ( "errors" - "net" "regexp" - "strings" "github.com/pilosa/pilosa/internal" ) @@ -108,26 +106,16 @@ func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" -// ValidateName ensures that the name is a valid format. -func ValidateName(name string) error { +// validateName ensures that the name is a valid format. +func validateName(name string) error { if !nameRegexp.Match([]byte(name)) { return ErrName } return nil } -// StringInSlice checks for substring a in the slice. -func StringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - -// StringSlicesAreEqual determines if two string slices are equal. -func StringSlicesAreEqual(a, b []string) bool { +// stringSlicesAreEqual determines if two string slices are equal. +func stringSlicesAreEqual(a, b []string) bool { if a == nil && b == nil { return true @@ -150,54 +138,6 @@ func StringSlicesAreEqual(a, b []string) bool { return true } -// SliceDiff returns the difference between two uint64 slices. -func SliceDiff(a, b []uint64) []uint64 { - m := make(map[uint64]uint64) - - for _, y := range b { - m[y]++ - } - - var ret []uint64 - for _, x := range a { - if m[x] > 0 { - m[x]-- - continue - } - ret = append(ret, x) - } - - return ret -} - -// ContainsSubstring checks to see if substring a is contained in any string in the slice. -func ContainsSubstring(a string, list []string) bool { - for _, b := range list { - if strings.Contains(b, a) { - return true - } - } - return false -} - -// HostToIP converts host to an IP4 address based on net.LookupIP(). -func HostToIP(host string) string { - // if host is not an IP addr, check net.LookupIP() - if net.ParseIP(host) == nil { - hosts, err := net.LookupIP(host) - if err != nil { - return host - } - for _, h := range hosts { - // this restricts pilosa to IP4 - if h.To4() != nil { - return h.String() - } - } - } - return host -} - // AddressWithDefaults converts addr into a valid address, // using defaults when necessary. func AddressWithDefaults(addr string) (*URI, error) { diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go new file mode 100644 index 000000000..139d5c2b7 --- /dev/null +++ b/pilosa_internal_test.go @@ -0,0 +1,43 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "testing" +) + +func TestValidateName(t *testing.T) { + names := []string{ + "a", "ab", "ab1", "b-c", "d_e", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } + for _, name := range names { + if validateName(name) != nil { + t.Fatalf("Should be valid index name: %s", name) + } + } +} + +func TestValidateNameInvalid(t *testing.T) { + names := []string{ + "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", + } + for _, name := range names { + if validateName(name) == nil { + t.Fatalf("Should be invalid index name: %s", name) + } + } +} diff --git a/pilosa_test.go b/pilosa_test.go index 41b0d7098..1c98686e5 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -22,54 +22,6 @@ import ( _ "github.com/pilosa/pilosa/test" ) -func TestValidateName(t *testing.T) { - names := []string{ - "a", "ab", "ab1", "b-c", "d_e", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - } - for _, name := range names { - if pilosa.ValidateName(name) != nil { - t.Fatalf("Should be valid index name: %s", name) - } - } -} - -func TestValidateNameInvalid(t *testing.T) { - names := []string{ - "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", - } - for _, name := range names { - if pilosa.ValidateName(name) == nil { - t.Fatalf("Should be invalid index name: %s", name) - } - } -} - -func TestStringInSlice(t *testing.T) { - list := []string{"localhost:10101", "localhost:10102", "localhost:10103"} - substr := "localhost:10101" - if !pilosa.StringInSlice(substr, list) { - t.Fatalf("Expected substring %s in %v", substr, list) - } - substr = "10101" - if pilosa.StringInSlice(substr, list) { - t.Fatalf("Expected substring %s not in %v", substr, list) - } -} - -func TestContainsSubstring(t *testing.T) { - list := []string{"localhost:10101", "localhost:10102", "localhost:10103"} - substr := "10101" - if !pilosa.ContainsSubstring(substr, list) { - t.Fatalf("Expected substring %s contained in %v", substr, list) - } - substr = "4000" - if pilosa.ContainsSubstring(substr, list) { - t.Fatalf("Expected substring %s in not contained in %v", substr, list) - } -} - func TestAddressWithDefaults(t *testing.T) { tests := []struct { addr string diff --git a/server.go b/server.go index f330b6e6a..c94bcbca0 100644 --- a/server.go +++ b/server.go @@ -659,7 +659,7 @@ func (s *Server) monitorDiagnostics() { // Flush the diagnostics metrics at startup, then on each tick interval flush := func() { - openFiles, err := CountOpenFiles() + openFiles, err := countOpenFiles() if err == nil { s.diagnostics.Set("OpenFiles", openFiles) } @@ -716,7 +716,7 @@ func (s *Server) monitorRuntime() { // Record the number of go routines. s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) - openFiles, err := CountOpenFiles() + openFiles, err := countOpenFiles() // Open File handles. if err == nil { s.Holder.Stats.Gauge("OpenFiles", float64(openFiles), 1.0) @@ -732,8 +732,8 @@ func (s *Server) monitorRuntime() { } } -// CountOpenFiles on operating systems that support lsof. -func CountOpenFiles() (int, error) { +// countOpenFiles on operating systems that support lsof. +func countOpenFiles() (int, error) { switch runtime.GOOS { case "darwin", "linux", "unix", "freebsd": // -b option avoid kernel blocks @@ -747,9 +747,9 @@ func CountOpenFiles() (int, error) { return len(lines), nil case "windows": // TODO: count open file handles on windows - return 0, errors.New("CountOpenFiles() on Windows is not supported") + return 0, errors.New("countOpenFiles() on Windows is not supported") default: - return 0, errors.New("CountOpenFiles() on this OS is not supported") + return 0, errors.New("countOpenFiles() on this OS is not supported") } } diff --git a/server/server_test.go b/server/server_test.go index a8cc2f8f4..971795156 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "math/rand" "reflect" - "runtime" "sort" "strings" "testing" @@ -263,21 +262,6 @@ func tempMkdir(t *testing.T) string { return dir } -// Ensure the file handle count is working -func TestCountOpenFiles(t *testing.T) { - // Windows is not supported yet - if runtime.GOOS == "windows" { - t.Skip("Skipping unsupported CountOpenFiles test on Windows.") - } - count, err := pilosa.CountOpenFiles() - if err != nil { - t.Errorf("CountOpenFiles failed: %s", err) - } - if count == 0 { - t.Error("CountOpenFiles returned invalid value 0.") - } -} - func TestMain_RecalculateHashes(t *testing.T) { const clusterSize = 5 cluster := test.MustRunMainWithCluster(t, clusterSize) diff --git a/server_internal_test.go b/server_internal_test.go new file mode 100644 index 000000000..e64f80a0f --- /dev/null +++ b/server_internal_test.go @@ -0,0 +1,35 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "runtime" + "testing" +) + +// Ensure the file handle count is working +func TestCountOpenFiles(t *testing.T) { + // Windows is not supported yet + if runtime.GOOS == "windows" { + t.Skip("Skipping unsupported countOpenFiles test on Windows.") + } + count, err := countOpenFiles() + if err != nil { + t.Errorf("countOpenFiles failed: %s", err) + } + if count == 0 { + t.Error("countOpenFiles returned invalid value 0.") + } +} diff --git a/server_test.go b/server_test.go index 6cbbe191e..2f1003592 100644 --- a/server_test.go +++ b/server_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/stats.go b/stats.go index 313708fbc..23c3e0bc8 100644 --- a/stats.go +++ b/stats.go @@ -110,7 +110,7 @@ func (c *ExpvarStatsClient) WithTags(tags ...string) StatsClient { return &ExpvarStatsClient{ m: m, - tags: UnionStringSlice(c.tags, tags), + tags: unionStringSlice(c.tags, tags), } } @@ -249,8 +249,8 @@ func (a MultiStatsClient) Close() error { return nil } -// UnionStringSlice returns a sorted set of tags which combine a & b. -func UnionStringSlice(a, b []string) []string { +// unionStringSlice returns a sorted set of tags which combine a & b. +func unionStringSlice(a, b []string) []string { // Sort both sets first. sort.Strings(a) sort.Strings(b) diff --git a/statsd/statsd.go b/statsd/statsd.go index 0cec63718..81cfbb2b4 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -15,6 +15,7 @@ package statsd import ( + "sort" "time" "github.com/DataDog/datadog-go/statsd" @@ -72,7 +73,7 @@ func (c *StatsClient) Tags() []string { func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient { return &StatsClient{ client: c.client, - tags: pilosa.UnionStringSlice(c.tags, tags), + tags: unionStringSlice(c.tags, tags), logger: c.logger, } } @@ -124,3 +125,38 @@ func (c *StatsClient) Timing(name string, value time.Duration, rate float64) { func (c *StatsClient) SetLogger(logger pilosa.Logger) { c.logger = logger } + +// unionStringSlice returns a sorted set of tags which combine a & b. +func unionStringSlice(a, b []string) []string { + // Sort both sets first. + sort.Strings(a) + sort.Strings(b) + + // Find size of largest slice. + n := len(a) + if len(b) > n { + n = len(b) + } + + // Exit if both sets are empty. + if n == 0 { + return nil + } + + // Iterate over both in order and merge. + other := make([]string, 0, n) + for len(a) > 0 || len(b) > 0 { + if len(a) == 0 { + other, b = append(other, b[0]), b[1:] + } else if len(b) == 0 { + other, a = append(other, a[0]), a[1:] + } else if a[0] < b[0] { + other, a = append(other, a[0]), a[1:] + } else if b[0] < a[0] { + other, b = append(other, b[0]), b[1:] + } else { + other, a, b = append(other, a[0]), a[1:], b[1:] + } + } + return other +} diff --git a/time.go b/time.go index 517292071..def889304 100644 --- a/time.go +++ b/time.go @@ -79,8 +79,8 @@ func ParseTimeQuantum(v string) (TimeQuantum, error) { return q, nil } -// ViewByTimeUnit returns the view name for time with a given quantum unit. -func ViewByTimeUnit(name string, t time.Time, unit rune) string { +// viewByTimeUnit returns the view name for time with a given quantum unit. +func viewByTimeUnit(name string, t time.Time, unit rune) string { switch unit { case 'Y': return fmt.Sprintf("%s_%s", name, t.Format("2006")) @@ -95,11 +95,11 @@ func ViewByTimeUnit(name string, t time.Time, unit rune) string { } } -// ViewsByTime returns a list of views for a given timestamp. -func ViewsByTime(name string, t time.Time, q TimeQuantum) []string { +// viewsByTime returns a list of views for a given timestamp. +func viewsByTime(name string, t time.Time, q TimeQuantum) []string { a := make([]string, 0, len(q)) for _, unit := range q { - view := ViewByTimeUnit(name, t, unit) + view := viewByTimeUnit(name, t, unit) if view == "" { continue } @@ -108,8 +108,8 @@ func ViewsByTime(name string, t time.Time, q TimeQuantum) []string { return a } -// ViewsByTimeRange returns a list of views to traverse to query a time range. -func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { +// viewsByTimeRange returns a list of views to traverse to query a time range. +func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { t := start // Save flags for performance. @@ -127,7 +127,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextDayGTE(t, end) { break } else if t.Hour() != 0 { - results = append(results, ViewByTimeUnit(name, t, 'H')) + results = append(results, viewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) continue } @@ -138,7 +138,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextMonthGTE(t, end) { break } else if t.Day() != 1 { - results = append(results, ViewByTimeUnit(name, t, 'D')) + results = append(results, viewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) continue } @@ -148,7 +148,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextYearGTE(t, end) { break } else if t.Month() != 1 { - results = append(results, ViewByTimeUnit(name, t, 'M')) + results = append(results, viewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) continue } @@ -164,16 +164,16 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string // Walk back down from largest units to smallest units. for t.Before(end) { if hasYear && nextYearGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'Y')) + results = append(results, viewByTimeUnit(name, t, 'Y')) t = t.AddDate(1, 0, 0) } else if hasMonth && nextMonthGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'M')) + results = append(results, viewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) } else if hasDay && nextDayGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'D')) + results = append(results, viewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) } else if hasHour { - results = append(results, ViewByTimeUnit(name, t, 'H')) + results = append(results, viewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) } else { break diff --git a/time_test.go b/time_internal_test.go similarity index 68% rename from time_test.go rename to time_internal_test.go index 3ab652455..2920685fc 100644 --- a/time_test.go +++ b/time_internal_test.go @@ -12,28 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package pilosa import ( "reflect" "testing" "time" - - "github.com/pilosa/pilosa" ) // Ensure string can be parsed into time quantum. func TestParseTimeQuantum(t *testing.T) { t.Run("OK", func(t *testing.T) { - if q, err := pilosa.ParseTimeQuantum("YMDH"); err != nil { + if q, err := ParseTimeQuantum("YMDH"); err != nil { t.Fatalf("unexpected error: %s", err) - } else if q != pilosa.TimeQuantum("YMDH") { + } else if q != TimeQuantum("YMDH") { t.Fatalf("unexpected quantum: %#v", q) } }) t.Run("ErrInvalidTimeQuantum", func(t *testing.T) { - if _, err := pilosa.ParseTimeQuantum("BADQUANTUM"); err != pilosa.ErrInvalidTimeQuantum { + if _, err := ParseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum { t.Fatalf("unexpected error: %s", err) } }) @@ -44,22 +42,22 @@ func TestViewByTimeUnit(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("Y", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'Y'); s != "F_2000" { + if s := viewByTimeUnit("F", ts, 'Y'); s != "F_2000" { t.Fatalf("unexpected name: %s", s) } }) t.Run("M", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'M'); s != "F_200001" { + if s := viewByTimeUnit("F", ts, 'M'); s != "F_200001" { t.Fatalf("unexpected name: %s", s) } }) t.Run("D", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'D'); s != "F_20000102" { + if s := viewByTimeUnit("F", ts, 'D'); s != "F_20000102" { t.Fatalf("unexpected name: %s", s) } }) t.Run("H", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { + if s := viewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { t.Fatalf("unexpected name: %s", s) } }) @@ -70,14 +68,14 @@ func TestViewsByTime(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("YMDH", func(t *testing.T) { - a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("YMDH")) + a := viewsByTime("F", ts, mustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) { t.Fatalf("unexpected names: %+v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("D")) + a := viewsByTime("F", ts, mustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000102"}) { t.Fatalf("unexpected names: %+v", a) } @@ -87,82 +85,82 @@ func TestViewsByTime(t *testing.T) { // Ensure sets of fields can be returned for a given time range. func TestViewsByTimeRange(t *testing.T) { t.Run("Y", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2002-01-01 00:00"), mustParseTimeQuantum("Y")) if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YM", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) + a := viewsByTimeRange("F", mustParseTime("2000-11-01 00:00"), mustParseTime("2003-03-01 00:00"), mustParseTimeQuantum("YM")) if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YMD", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) + a := viewsByTimeRange("F", mustParseTime("2000-11-28 00:00"), mustParseTime("2003-03-02 00:00"), mustParseTimeQuantum("YMD")) if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YMDH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) + a := viewsByTimeRange("F", mustParseTime("2000-11-28 22:00"), mustParseTime("2002-03-01 03:00"), mustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("M", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-03-01 00:00"), mustParseTimeQuantum("M")) if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("MD", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) + a := viewsByTimeRange("F", mustParseTime("2000-11-29 00:00"), mustParseTime("2002-02-03 00:00"), mustParseTimeQuantum("MD")) if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("MDH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) + a := viewsByTimeRange("F", mustParseTime("2000-11-29 22:00"), mustParseTime("2002-03-02 03:00"), mustParseTimeQuantum("MDH")) if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-04 00:00"), mustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("DH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 22:00"), mustParseTime("2000-03-01 02:00"), mustParseTimeQuantum("DH")) if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("H", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-01 02:00"), mustParseTimeQuantum("H")) if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) { t.Fatalf("unexpected fields: %#v", a) } }) } -// DefaultTimeLayout is the time layout used by the tests. -const DefaultTimeLayout = "2006-01-02 15:04" +// defaultTimeLayout is the time layout used by the tests. +const defaultTimeLayout = "2006-01-02 15:04" -// MustParseTime parses value using DefaultTimeLayout. Panic on error. -func MustParseTime(value string) time.Time { - v, err := time.Parse(DefaultTimeLayout, value) +// mustParseTime parses value using DefaultTimeLayout. Panic on error. +func mustParseTime(value string) time.Time { + v, err := time.Parse(defaultTimeLayout, value) if err != nil { panic(err) } return v } -// MustParseTimeQuantum parses v into a time quantum. Panic on error. -func MustParseTimeQuantum(v string) pilosa.TimeQuantum { - q, err := pilosa.ParseTimeQuantum(v) +// mustParseTimeQuantum parses v into a time quantum. Panic on error. +func mustParseTimeQuantum(v string) TimeQuantum { + q, err := ParseTimeQuantum(v) if err != nil { panic(err) } diff --git a/view.go b/view.go index a7aaf0cb8..baef603d3 100644 --- a/view.go +++ b/view.go @@ -34,8 +34,8 @@ const ( viewBSIGroupPrefix = "bsig_" ) -// IsValidView returns true if name is valid. -func IsValidView(name string) bool { +// isValidView returns true if name is valid. +func isValidView(name string) bool { return name == ViewStandard } From fba865fc6cc5c76d21cd8af21705617f19b114ab Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 13 Jun 2018 11:24:25 -0500 Subject: [PATCH 68/78] Remove more net/http references --- api.go | 2 -- cluster.go | 4 ---- fragment.go | 6 ++---- handler.go | 6 +++--- holder.go | 15 ++++++--------- holder_test.go | 10 ++++------ http/handler.go | 12 ++++++++++++ server.go | 26 ++------------------------ server/server.go | 2 -- 9 files changed, 29 insertions(+), 54 deletions(-) diff --git a/api.go b/api.go index 36af91b0f..ab4948588 100644 --- a/api.go +++ b/api.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "strconv" "strings" "time" @@ -45,7 +44,6 @@ type API struct { BroadcastHandler BroadcastHandler StatusHandler StatusHandler Cluster *Cluster - RemoteClient *http.Client Logger Logger } diff --git a/cluster.go b/cluster.go index 28feb80a3..dccac02a1 100644 --- a/cluster.go +++ b/cluster.go @@ -21,7 +21,6 @@ import ( "hash/fnv" "io/ioutil" "math/rand" - "net/http" "os" "path/filepath" "sort" @@ -264,9 +263,6 @@ type Cluster struct { Logger Logger - // - RemoteClient *http.Client - InternalClient InternalClient } diff --git a/fragment.go b/fragment.go index 012c62503..ab50117fc 100644 --- a/fragment.go +++ b/fragment.go @@ -25,7 +25,6 @@ import ( "hash" "io" "io/ioutil" - "net/http" "os" "sort" "sync" @@ -1719,9 +1718,8 @@ func (h *blockHasher) WriteValue(v uint64) { type FragmentSyncer struct { Fragment *Fragment - Node *Node - Cluster *Cluster - RemoteClient *http.Client + Node *Node + Cluster *Cluster Closing <-chan struct{} } diff --git a/handler.go b/handler.go index f9c3435af..7c2b76a3f 100644 --- a/handler.go +++ b/handler.go @@ -2,7 +2,7 @@ package pilosa import ( "encoding/json" - "net/http" + "net" ) // QueryRequest represent a request to process a query. @@ -61,13 +61,13 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { } type Handler interface { - http.Handler + Serve(ln net.Listener, closing <-chan struct{}) GetAPI() *API } type NopHandler struct{} -func (n *NopHandler) ServeHTTP(_ http.ResponseWriter, _ *http.Request) {} +func (n *NopHandler) Serve(ln net.Listener, closing <-chan struct{}) {} func (n *NopHandler) GetAPI() *API { return nil diff --git a/holder.go b/holder.go index e7af65643..8dac3329b 100644 --- a/holder.go +++ b/holder.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io/ioutil" - "net/http" "os" "path" "path/filepath" @@ -563,9 +562,8 @@ func (h *Holder) logStartup() error { type HolderSyncer struct { Holder *Holder - Node *Node - Cluster *Cluster - RemoteClient *http.Client + Node *Node + Cluster *Cluster // Stats Stats StatsClient @@ -755,11 +753,10 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err // Sync fragments together. fs := FragmentSyncer{ - Fragment: frag, - Node: s.Node, - Cluster: s.Cluster, - Closing: s.Closing, - RemoteClient: s.RemoteClient, + Fragment: frag, + Node: s.Node, + Cluster: s.Cluster, + Closing: s.Closing, } if err := fs.syncFragment(); err != nil { return errors.Wrap(err, "syncing fragment") diff --git a/holder_test.go b/holder_test.go index 31b50b900..72a10b815 100644 --- a/holder_test.go +++ b/holder_test.go @@ -362,7 +362,6 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { client := http.GetHTTPClient(nil) httpClient := http.NewInternalClientFromURI(uri, client) cluster.InternalClient = httpClient - cluster.RemoteClient = client // Create a local holder. hldr0 := test.MustOpenHolder() @@ -419,11 +418,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Set up syncer. syncer := pilosa.HolderSyncer{ - Holder: hldr0.Holder, - Node: cluster.Nodes[0], - Cluster: cluster, - RemoteClient: http.GetHTTPClient(nil), - Stats: pilosa.NopStatsClient, + Holder: hldr0.Holder, + Node: cluster.Nodes[0], + Cluster: cluster, + Stats: pilosa.NopStatsClient, } if err := syncer.SyncHolder(); err != nil { diff --git a/http/handler.go b/http/handler.go index df5108826..41a5faf55 100644 --- a/http/handler.go +++ b/http/handler.go @@ -117,6 +117,18 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) { return handler, nil } +func (h *Handler) Serve(ln net.Listener, closing <-chan struct{}) { + server := &http.Server{Handler: h} + go func() { + <-closing + server.Close() + }() + err := server.Serve(ln) + if err != nil && err.Error() != "http: Server closed" { + h.Logger.Printf("HTTP handler terminated with error: %s\n", err) + } +} + func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") diff --git a/server.go b/server.go index f330b6e6a..b1ce4bfaa 100644 --- a/server.go +++ b/server.go @@ -19,7 +19,6 @@ import ( "fmt" "log" "net" - "net/http" "os" "os/exec" "path/filepath" @@ -63,7 +62,6 @@ type Server struct { Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver Gossiper Gossiper - remoteClient *http.Client systemInfo SystemInfo gcNotifier GCNotifier NewAttrStore func(string) AttrStore @@ -162,15 +160,6 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { } } -// TODO: Remove RemoteClient -func OptServerRemoteClient(c *http.Client) ServerOption { - return func(s *Server) error { - s.remoteClient = c - s.Cluster.RemoteClient = c - return nil - } -} - func OptServerInternalClient(c InternalClient) ServerOption { return func(s *Server) error { s.executor = NewExecutor(OptExecutorInternalQueryClient(c)) @@ -313,18 +302,8 @@ func (s *Server) Open() error { // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster - // Serve HTTP. - go func() { - server := &http.Server{Handler: s.handler} - go func() { - <-s.closing - server.Close() - }() - err := server.Serve(s.ln) - if err != nil && err.Error() != "http: Server closed" { - s.logger.Printf("HTTP handler terminated with error: %s\n", err) - } - }() + // Serve handler. + go s.handler.Serve(s.ln, s.closing) // Start the BroadcastReceiver. if err := s.BroadcastReceiver.Start(s); err != nil { @@ -424,7 +403,6 @@ func (s *Server) monitorAntiEntropy() { syncer.Node = s.Cluster.Node syncer.Cluster = s.Cluster syncer.Closing = s.closing - syncer.RemoteClient = s.remoteClient syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer") // Sync holders. diff --git a/server/server.go b/server/server.go index 42b547d43..d034a4aa8 100644 --- a/server/server.go +++ b/server/server.go @@ -216,7 +216,6 @@ func (m *Command) SetupServer() error { } c := http.GetHTTPClient(TLSConfig) - api.RemoteClient = c m.Server, err = pilosa.NewServer( pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), @@ -235,7 +234,6 @@ func (m *Command) SetupServer() error { pilosa.OptServerStatsClient(statsClient), pilosa.OptServerListener(ln), pilosa.OptServerURI(uri), - pilosa.OptServerRemoteClient(c), pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), ) From 7d91261968f8bc9a3165da581100e1ae4b0a1f23 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 11 Jun 2018 17:54:31 -0500 Subject: [PATCH 69/78] un-export some package level constants --- attr.go | 26 +++++------ broadcast.go | 90 +++++++++++++++++++-------------------- cache.go | 6 +-- executor.go | 16 +++---- field.go | 8 ++-- fragment.go | 30 ++++++------- fragment_internal_test.go | 2 +- holder.go | 6 +-- view.go | 2 +- view_internal_test.go | 2 +- 10 files changed, 93 insertions(+), 95 deletions(-) diff --git a/attr.go b/attr.go index 168ac0052..e73f303af 100644 --- a/attr.go +++ b/attr.go @@ -24,10 +24,10 @@ import ( // Attribute data type enum. const ( - AttrTypeString = 1 - AttrTypeInt = 2 - AttrTypeBool = 3 - AttrTypeFloat = 4 + attrTypeString = 1 + attrTypeInt = 2 + attrTypeBool = 3 + attrTypeFloat = 4 ) // AttrStore represents an interface for handling row/column attributes. @@ -165,19 +165,19 @@ func encodeAttr(key string, value interface{}) *internal.Attr { pb := &internal.Attr{Key: key} switch value := value.(type) { case string: - pb.Type = AttrTypeString + pb.Type = attrTypeString pb.StringValue = value case float64: - pb.Type = AttrTypeFloat + pb.Type = attrTypeFloat pb.FloatValue = value case uint64: - pb.Type = AttrTypeInt + pb.Type = attrTypeInt pb.IntValue = int64(value) case int64: - pb.Type = AttrTypeInt + pb.Type = attrTypeInt pb.IntValue = value case bool: - pb.Type = AttrTypeBool + pb.Type = attrTypeBool pb.BoolValue = value } return pb @@ -186,13 +186,13 @@ func encodeAttr(key string, value interface{}) *internal.Attr { // decodeAttr converts from an Attr internal representation to a key/value pair. func decodeAttr(attr *internal.Attr) (key string, value interface{}) { switch attr.Type { - case AttrTypeString: + case attrTypeString: return attr.Key, attr.StringValue - case AttrTypeInt: + case attrTypeInt: return attr.Key, attr.IntValue - case AttrTypeBool: + case attrTypeBool: return attr.Key, attr.BoolValue - case AttrTypeFloat: + case attrTypeFloat: return attr.Key, attr.FloatValue default: return attr.Key, nil diff --git a/broadcast.go b/broadcast.go index 6b26b5d58..9b894fea2 100644 --- a/broadcast.go +++ b/broadcast.go @@ -120,21 +120,21 @@ func (n *nopGossiper) SendAsync(pb proto.Message) error { // Broadcast message types. const ( - MessageTypeCreateSlice = iota - MessageTypeCreateIndex - MessageTypeDeleteIndex - MessageTypeCreateField - MessageTypeDeleteField - MessageTypeCreateView - MessageTypeDeleteView - MessageTypeClusterStatus - MessageTypeResizeInstruction - MessageTypeResizeInstructionComplete - MessageTypeSetCoordinator - MessageTypeUpdateCoordinator - MessageTypeNodeState - MessageTypeRecalculateCaches - MessageTypeNodeEvent + messageTypeCreateSlice = iota + messageTypeCreateIndex + messageTypeDeleteIndex + messageTypeCreateField + messageTypeDeleteField + messageTypeCreateView + messageTypeDeleteView + messageTypeClusterStatus + messageTypeResizeInstruction + messageTypeResizeInstructionComplete + messageTypeSetCoordinator + messageTypeUpdateCoordinator + messageTypeNodeState + messageTypeRecalculateCaches + messageTypeNodeEvent ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -142,35 +142,35 @@ func MarshalMessage(m proto.Message) ([]byte, error) { var typ uint8 switch obj := m.(type) { case *internal.CreateSliceMessage: - typ = MessageTypeCreateSlice + typ = messageTypeCreateSlice case *internal.CreateIndexMessage: - typ = MessageTypeCreateIndex + typ = messageTypeCreateIndex case *internal.DeleteIndexMessage: - typ = MessageTypeDeleteIndex + typ = messageTypeDeleteIndex case *internal.CreateFieldMessage: - typ = MessageTypeCreateField + typ = messageTypeCreateField case *internal.DeleteFieldMessage: - typ = MessageTypeDeleteField + typ = messageTypeDeleteField case *internal.CreateViewMessage: - typ = MessageTypeCreateView + typ = messageTypeCreateView case *internal.DeleteViewMessage: - typ = MessageTypeDeleteView + typ = messageTypeDeleteView case *internal.ClusterStatus: - typ = MessageTypeClusterStatus + typ = messageTypeClusterStatus case *internal.ResizeInstruction: - typ = MessageTypeResizeInstruction + typ = messageTypeResizeInstruction case *internal.ResizeInstructionComplete: - typ = MessageTypeResizeInstructionComplete + typ = messageTypeResizeInstructionComplete case *internal.SetCoordinatorMessage: - typ = MessageTypeSetCoordinator + typ = messageTypeSetCoordinator case *internal.UpdateCoordinatorMessage: - typ = MessageTypeUpdateCoordinator + typ = messageTypeUpdateCoordinator case *internal.NodeStateMessage: - typ = MessageTypeNodeState + typ = messageTypeNodeState case *internal.RecalculateCaches: - typ = MessageTypeRecalculateCaches + typ = messageTypeRecalculateCaches case *internal.NodeEventMessage: - typ = MessageTypeNodeEvent + typ = messageTypeNodeEvent default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -187,35 +187,35 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { var m proto.Message switch typ { - case MessageTypeCreateSlice: + case messageTypeCreateSlice: m = &internal.CreateSliceMessage{} - case MessageTypeCreateIndex: + case messageTypeCreateIndex: m = &internal.CreateIndexMessage{} - case MessageTypeDeleteIndex: + case messageTypeDeleteIndex: m = &internal.DeleteIndexMessage{} - case MessageTypeCreateField: + case messageTypeCreateField: m = &internal.CreateFieldMessage{} - case MessageTypeDeleteField: + case messageTypeDeleteField: m = &internal.DeleteFieldMessage{} - case MessageTypeCreateView: + case messageTypeCreateView: m = &internal.CreateViewMessage{} - case MessageTypeDeleteView: + case messageTypeDeleteView: m = &internal.DeleteViewMessage{} - case MessageTypeClusterStatus: + case messageTypeClusterStatus: m = &internal.ClusterStatus{} - case MessageTypeResizeInstruction: + case messageTypeResizeInstruction: m = &internal.ResizeInstruction{} - case MessageTypeResizeInstructionComplete: + case messageTypeResizeInstructionComplete: m = &internal.ResizeInstructionComplete{} - case MessageTypeSetCoordinator: + case messageTypeSetCoordinator: m = &internal.SetCoordinatorMessage{} - case MessageTypeUpdateCoordinator: + case messageTypeUpdateCoordinator: m = &internal.UpdateCoordinatorMessage{} - case MessageTypeNodeState: + case messageTypeNodeState: m = &internal.NodeStateMessage{} - case MessageTypeRecalculateCaches: + case messageTypeRecalculateCaches: m = &internal.RecalculateCaches{} - case MessageTypeNodeEvent: + case messageTypeNodeEvent: m = &internal.NodeEventMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) diff --git a/cache.go b/cache.go index ec9ade91b..e6a7dc283 100644 --- a/cache.go +++ b/cache.go @@ -27,8 +27,8 @@ import ( ) const ( - // ThresholdFactor is used to calculate the threshold for new items entering the cache - ThresholdFactor = 1.1 + // thresholdFactor is used to calculate the threshold for new items entering the cache + thresholdFactor = 1.1 ) // Cache represents a cache of counts. @@ -158,7 +158,7 @@ type RankCache struct { func NewRankCache(maxEntries uint32) *RankCache { return &RankCache{ maxEntries: maxEntries, - thresholdBuffer: int(ThresholdFactor * float64(maxEntries)), + thresholdBuffer: int(thresholdFactor * float64(maxEntries)), entries: make(map[uint64]uint64), stats: NopStatsClient, } diff --git a/executor.go b/executor.go index 03c636215..fd07bd82a 100644 --- a/executor.go +++ b/executor.go @@ -25,13 +25,13 @@ import ( "github.com/pkg/errors" ) -// DefaultField is the field used if one is not specified. +// defaultField is the field used if one is not specified. const ( - DefaultField = "general" + defaultField = "general" - // MinThreshold is the lowest count to use in a Top-N operation when + // defaultMinThreshold is the lowest count to use in a Top-N operation when // looking for additional id/count pairs. - MinThreshold = 1 + defaultMinThreshold = 1 columnLabel = "col" rowLabel = "row" @@ -588,7 +588,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca // Set default field. if field == "" { - field = DefaultField + field = defaultField } f := e.Holder.Fragment(index, field, ViewStandard, slice) @@ -597,7 +597,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca } if minThreshold <= 0 { - minThreshold = MinThreshold + minThreshold = defaultMinThreshold } if tanimotoThreshold > 100 { @@ -646,7 +646,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. // Fetch field & row label based on argument. field, _ := c.Args["field"].(string) if field == "" { - field = DefaultField + field = defaultField } f := e.Holder.Field(index, field) if f == nil { @@ -700,7 +700,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Parse field, use default if unset. field, _ := c.Args["field"].(string) if field == "" { - field = DefaultField + field = defaultField } // Retrieve column label. diff --git a/field.go b/field.go index d03da2c22..2c7598037 100644 --- a/field.go +++ b/field.go @@ -33,10 +33,10 @@ import ( const ( DefaultFieldType = FieldTypeSet - DefaultCacheType = CacheTypeRanked + defaultCacheType = CacheTypeRanked // Default ranked field cache - DefaultCacheSize = 50000 + defaultCacheSize = 50000 ) // Field types. @@ -101,8 +101,8 @@ func NewField(path, index, name string, opts ...FieldOption) (*Field, error) { options: FieldOptions{ Type: DefaultFieldType, - CacheType: DefaultCacheType, - CacheSize: DefaultCacheSize, + CacheType: defaultCacheType, + CacheSize: defaultCacheSize, }, Logger: NopLogger, diff --git a/fragment.go b/fragment.go index 012c62503..d06676170 100644 --- a/fragment.go +++ b/fragment.go @@ -48,22 +48,20 @@ const ( // SliceWidth is the number of column IDs in a slice. SliceWidth = 1048576 - // SnapshotExt is the file extension used for an in-process snapshot. - SnapshotExt = ".snapshotting" + // snapshotExt is the file extension used for an in-process snapshot. + snapshotExt = ".snapshotting" - // CopyExt is the file extension used for the temp file used while copying. - CopyExt = ".copying" + // copyExt is the file extension used for the temp file used while copying. + copyExt = ".copying" - // CacheExt is the file extension for persisted cache ids. - CacheExt = ".cache" + // cacheExt is the file extension for persisted cache ids. + cacheExt = ".cache" // HashBlockSize is the number of rows in a merkle hash block. HashBlockSize = 100 -) -const ( - // DefaultFragmentMaxOpN is the default value for Fragment.MaxOpN. - DefaultFragmentMaxOpN = 2000 + // defaultFragmentMaxOpN is the default value for Fragment.MaxOpN. + defaultFragmentMaxOpN = 2000 ) // Fragment represents the intersection of a field and slice in an index. @@ -120,18 +118,18 @@ func NewFragment(path, index, field, view string, slice uint64) *Fragment { field: field, view: view, slice: slice, - CacheType: DefaultCacheType, - CacheSize: DefaultCacheSize, + CacheType: defaultCacheType, + CacheSize: defaultCacheSize, Logger: NopLogger, - MaxOpN: DefaultFragmentMaxOpN, + MaxOpN: defaultFragmentMaxOpN, stats: NopStatsClient, } } // cachePath returns the path to the fragment's cache data. -func (f *Fragment) cachePath() string { return f.path + CacheExt } +func (f *Fragment) cachePath() string { return f.path + cacheExt } // Open opens the underlying storage. func (f *Fragment) Open() error { @@ -1432,7 +1430,7 @@ func (f *Fragment) snapshot() error { defer track(start, completeMessage, f.stats, f.Logger) // Create a temporary file to snapshot to. - snapshotPath := f.path + SnapshotExt + snapshotPath := f.path + snapshotExt file, err := os.Create(snapshotPath) if err != nil { return fmt.Errorf("create snapshot file: %s", err) @@ -1636,7 +1634,7 @@ func (f *Fragment) ReadFrom(r io.Reader) (n int64, err error) { func (f *Fragment) readStorageFromArchive(r io.Reader) error { // Create a temporary file to copy into. - path := f.path + CopyExt + path := f.path + copyExt file, err := os.Create(path) if err != nil { return errors.Wrap(err, "creating directory") diff --git a/fragment_internal_test.go b/fragment_internal_test.go index e7a77dafb..6c733ba4c 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1245,7 +1245,7 @@ func mustOpenFragment(index, field, view string, slice uint64, cacheType string) file.Close() if cacheType == "" { - cacheType = DefaultCacheType + cacheType = defaultCacheType } f := NewFragment(file.Name(), index, field, view, slice) diff --git a/holder.go b/holder.go index e7af65643..cdffd9c8d 100644 --- a/holder.go +++ b/holder.go @@ -34,8 +34,8 @@ import ( ) const ( - // DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. - DefaultCacheFlushInterval = 1 * time.Minute + // defaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. + defaultCacheFlushInterval = 1 * time.Minute // FileLimit is the maximum open file limit (ulimit -n) to automatically set. FileLimit = 262144 // (512^2) @@ -84,7 +84,7 @@ func NewHolder() *Holder { NewAttrStore: NewNopAttrStore, - CacheFlushInterval: DefaultCacheFlushInterval, + CacheFlushInterval: defaultCacheFlushInterval, Logger: NopLogger, } diff --git a/view.go b/view.go index baef603d3..b21c3f15a 100644 --- a/view.go +++ b/view.go @@ -73,7 +73,7 @@ func NewView(path, index, field, name string, cacheSize uint32) *View { name: name, cacheSize: cacheSize, - cacheType: DefaultCacheType, + cacheType: defaultCacheType, fragments: make(map[uint64]*Fragment), broadcaster: NopBroadcaster, diff --git a/view_internal_test.go b/view_internal_test.go index d0e8bfdd1..48df030db 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -26,7 +26,7 @@ func mustOpenView(index, field, name string) *View { panic(err) } - v := NewView(path, index, field, name, DefaultCacheSize) + v := NewView(path, index, field, name, defaultCacheSize) if err := v.open(); err != nil { panic(err) } From b36b3b16ffe398ca01f7fb98022edfc79a56b26e Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 14 Jun 2018 09:12:05 -0500 Subject: [PATCH 70/78] moved to pilosa core --- test/querygenerator_test.go => querygenerator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename test/querygenerator_test.go => querygenerator_test.go (99%) diff --git a/test/querygenerator_test.go b/querygenerator_test.go similarity index 99% rename from test/querygenerator_test.go rename to querygenerator_test.go index 4702e6b91..4641f5ed0 100644 --- a/test/querygenerator_test.go +++ b/querygenerator_test.go @@ -1,4 +1,4 @@ -package test +package pilosa_test import ( "fmt" From aa218905037e77c12383ca7b81fa78117f946a9f Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 14 Jun 2018 10:02:36 -0500 Subject: [PATCH 71/78] moved to internal/test --- internal/test/querygenerator.go | 190 ++++++++++++++++++ .../test/querygenerator_test.go | 186 +---------------- 2 files changed, 191 insertions(+), 185 deletions(-) create mode 100644 internal/test/querygenerator.go rename querygenerator_test.go => internal/test/querygenerator_test.go (57%) diff --git a/internal/test/querygenerator.go b/internal/test/querygenerator.go new file mode 100644 index 000000000..32720cb68 --- /dev/null +++ b/internal/test/querygenerator.go @@ -0,0 +1,190 @@ +package test + +import ( + "fmt" + "strconv" + "strings" + + "github.com/pilosa/pilosa/pql" +) + +type Args map[string]interface{} + +type Calls []*pql.Call + +func PQL(calls ...*pql.Call) *pql.Query { + return &pql.Query{Calls: calls} +} + +func Row(frame string, row int) *pql.Call { + return &pql.Call{ + Name: "Row", + Args: Args{ + "frame": frame, + "row": row, + }, + } +} + +func mutationArgs(args ...interface{}) Args { + rargs := make(Args) + for _, arg := range args { + switch v := arg.(type) { + case int: + rargs["column"] = v + case string: + if strings.Contains(v, "=") { + parts := strings.Split(v, "=") + rargs["frame"] = parts[0] + i, _ := strconv.ParseInt(parts[1], 10, 64) + rargs["value"] = i + } else { + rargs["timestamp"] = v + } + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs +} + +func Set(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Set", Args: mutationArgs(args...)} +} + +func Clear(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Clear", Args: mutationArgs(args...)} +} + +func magic(args ...interface{}) (Args, Calls) { + var ( + rargs Args + calls Calls + ) + + for _, arg := range args { + switch v := arg.(type) { + case Args: + rargs = v + case []*pql.Call: + calls = append(calls, v...) + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs, calls +} +func Count(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Count", Args: kvargs, Children: children} +} + +func Union(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Union", Args: kvargs, Children: children} +} + +func Intersect(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Intersect", Args: kvargs, Children: children} +} + +func Difference(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Difference", Args: kvargs, Children: children} +} +func Xor(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Xor", Args: kvargs, Children: children} +} + +func Between(frame string, min, max int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.BETWEEN, + "Value": []int{min, max}, + }, + } +} +func Lt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LT, + "Value": column, + }, + } +} +func Lte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LTE, + "Value": column, + }, + } +} +func Gt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GT, + "Value": column, + }, + } +} + +func Gte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GTE, + "Value": column, + }, + } +} +func CompareCall(a, b *pql.Call) bool { + if a.Name != b.Name { + return false + } + for k, i := range a.Args { + switch v := i.(type) { + case []int: + bside := b.Args[k] + for j := range v { + if v[j] != bside.([]int)[j] { + return false + } + + } + default: + if b.Args[k] != i { + return false + } + } + } + + if len(a.Children) == len(b.Children) { + for i := range a.Children { + if !CompareCall(a.Children[i], b.Children[i]) { + return false + } + } + } else { + return false + } + return true +} + +func Compare(a, b *pql.Query) bool { + for i := range a.Calls { + if !CompareCall(a.Calls[i], b.Calls[i]) { + return false + } + + } + return true +} diff --git a/querygenerator_test.go b/internal/test/querygenerator_test.go similarity index 57% rename from querygenerator_test.go rename to internal/test/querygenerator_test.go index 4641f5ed0..7af764602 100644 --- a/querygenerator_test.go +++ b/internal/test/querygenerator_test.go @@ -1,194 +1,10 @@ -package pilosa_test +package test import ( - "fmt" - "strconv" - "strings" "testing" - "github.com/pilosa/pilosa/pql" ) -type Args map[string]interface{} - -type Calls []*pql.Call - -func PQL(calls ...*pql.Call) *pql.Query { - return &pql.Query{Calls: calls} -} - -func Row(frame string, row int) *pql.Call { - return &pql.Call{ - Name: "Row", - Args: Args{ - "frame": frame, - "row": row, - }, - } -} - -func mutationArgs(args ...interface{}) Args { - rargs := make(Args) - for _, arg := range args { - switch v := arg.(type) { - case int: - rargs["column"] = v - case string: - if strings.Contains(v, "=") { - parts := strings.Split(v, "=") - rargs["frame"] = parts[0] - i, _ := strconv.ParseInt(parts[1], 10, 64) - rargs["value"] = i - } else { - rargs["timestamp"] = v - } - default: - fmt.Printf("wat %T!\n", v) - } - } - - return rargs -} - -func Set(args ...interface{}) *pql.Call { - return &pql.Call{Name: "Set", Args: mutationArgs(args...)} -} - -func Clear(args ...interface{}) *pql.Call { - return &pql.Call{Name: "Clear", Args: mutationArgs(args...)} -} - -func magic(args ...interface{}) (Args, Calls) { - var ( - rargs Args - calls Calls - ) - - for _, arg := range args { - switch v := arg.(type) { - case Args: - rargs = v - case []*pql.Call: - calls = append(calls, v...) - default: - fmt.Printf("wat %T!\n", v) - } - } - - return rargs, calls -} -func Count(args ...*pql.Call) *pql.Call { - kvargs, children := magic(args) - return &pql.Call{Name: "Count", Args: kvargs, Children: children} -} - -func Union(args ...*pql.Call) *pql.Call { - kvargs, children := magic(args) - return &pql.Call{Name: "Union", Args: kvargs, Children: children} -} - -func Intersect(args ...*pql.Call) *pql.Call { - kvargs, children := magic(args) - return &pql.Call{Name: "Intersect", Args: kvargs, Children: children} -} - -func Difference(args ...*pql.Call) *pql.Call { - kvargs, children := magic(args) - return &pql.Call{Name: "Difference", Args: kvargs, Children: children} -} -func Xor(args ...*pql.Call) *pql.Call { - kvargs, children := magic(args) - return &pql.Call{Name: "Xor", Args: kvargs, Children: children} -} - -func Between(frame string, min, max int) *pql.Call { - return &pql.Call{ - Name: "Range", - Args: Args{ - "Op": pql.BETWEEN, - "Value": []int{min, max}, - }, - } -} -func Lt(frame string, column int) *pql.Call { - return &pql.Call{ - Name: "Range", - Args: Args{ - "Op": pql.LT, - "Value": column, - }, - } -} -func Lte(frame string, column int) *pql.Call { - return &pql.Call{ - Name: "Range", - Args: Args{ - "Op": pql.LTE, - "Value": column, - }, - } -} -func Gt(frame string, column int) *pql.Call { - return &pql.Call{ - Name: "Range", - Args: Args{ - "Op": pql.GT, - "Value": column, - }, - } -} - -func Gte(frame string, column int) *pql.Call { - return &pql.Call{ - Name: "Range", - Args: Args{ - "Op": pql.GTE, - "Value": column, - }, - } -} -func CompareCall(a, b *pql.Call) bool { - if a.Name != b.Name { - return false - } - for k, i := range a.Args { - switch v := i.(type) { - case []int: - bside := b.Args[k] - for j := range v { - if v[j] != bside.([]int)[j] { - return false - } - - } - default: - if b.Args[k] != i { - return false - } - } - } - - if len(a.Children) == len(b.Children) { - for i := range a.Children { - if !CompareCall(a.Children[i], b.Children[i]) { - return false - } - } - } else { - return false - } - return true -} - -func Compare(a, b *pql.Query) bool { - for i := range a.Calls { - if !CompareCall(a.Calls[i], b.Calls[i]) { - return false - } - - } - return true -} func TestPQL_Generator(t *testing.T) { t.Run("pql.Query generator", func(t *testing.T) { From 060254e0d60b9a50aa757c75b94e1ecc43e6e433 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 15 Jun 2018 16:44:56 -0600 Subject: [PATCH 72/78] Key-to-ID Translation This commit adds id-to-key translation to make it easier for users to provide non-integer identifiers for rows & columns. --- Gopkg.lock | 14 +- api.go | 13 + ctl/server.go | 3 + executor.go | 112 +++++ executor_test.go | 126 ++++- field.go | 14 + holder.go | 8 +- http/handler.go | 52 ++ http/translator.go | 87 ++++ http/translator_test.go | 134 +++++ index.go | 22 +- inmem/translator.go | 215 ++++++++ inmem/translator_test.go | 132 +++++ internal/private.pb.go | 349 +++++++------ internal/private.proto | 2 + internal/public.pb.go | 35 +- mock/mock.go | 14 + mock/translator.go | 38 ++ pilosa.go | 3 + pql/ast.go | 33 ++ row.go | 10 + server.go | 31 +- server/config.go | 5 + server/server.go | 7 + statik/statik.go | 10 + test/executor.go | 2 + translate.go | 1006 ++++++++++++++++++++++++++++++++++++++ translate_test.go | 565 +++++++++++++++++++++ 28 files changed, 2829 insertions(+), 213 deletions(-) create mode 100644 http/translator.go create mode 100644 http/translator_test.go create mode 100644 inmem/translator.go create mode 100644 inmem/translator_test.go create mode 100644 mock/mock.go create mode 100644 mock/translator.go create mode 100644 statik/statik.go create mode 100644 translate.go create mode 100644 translate_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 0b4a8e9ea..b3fc9be2b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -70,6 +70,18 @@ packages = ["proto"] revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9" +[[projects]] + name = "github.com/google/go-cmp" + packages = [ + "cmp", + "cmp/cmpopts", + "cmp/internal/diff", + "cmp/internal/function", + "cmp/internal/value" + ] + revision = "3af367b6b30c263d47e8895973edcca9a49cf029" + version = "v0.2.0" + [[projects]] name = "github.com/gorilla/context" packages = ["."] @@ -304,6 +316,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "325d0fb217ec7f1509186ff947e184f6c8e65941f06000eb110180e65816b1a4" + inputs-digest = "40bd9c0a1a403580ad77f9ae84e81a97da1d1622b3f620bd000271c52b50b8b5" solver-name = "gps-cdcl" solver-version = 1 diff --git a/api.go b/api.go index ab4948588..7adff1d1b 100644 --- a/api.go +++ b/api.go @@ -44,6 +44,7 @@ type API struct { BroadcastHandler BroadcastHandler StatusHandler StatusHandler Cluster *Cluster + TranslateStore TranslateStore Logger Logger } @@ -124,6 +125,18 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er if err != nil { return resp, errors.Wrap(err, "reading column attrs") } + + // Translate column attributes, if necessary. + if api.TranslateStore != nil { + for _, col := range resp.ColumnAttrSets { + v, err := api.TranslateStore.TranslateColumnToString(req.Index, col.ID) + if err != nil { + return resp, err + } + col.Key, col.ID = v, 0 + } + } + resp.ColumnAttrSets = columnAttrSets } return resp, nil diff --git a/ctl/server.go b/ctl/server.go index a4c272a46..ad1ac5df5 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -43,6 +43,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") + // Translation + flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "URL for primary translation node for replication.") + // Gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.") diff --git a/executor.go b/executor.go index fd07bd82a..a99753023 100644 --- a/executor.go +++ b/executor.go @@ -50,6 +50,9 @@ type Executor struct { // Maximum number of SetBit() or ClearBit() commands per request. MaxWritesPerRequest int + + // Stores key/id translation data. + TranslateStore TranslateStore } // ExecutorOption is a functional option type for pilosa.Executor @@ -83,6 +86,11 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic return nil, ErrIndexRequired } + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + // Verify that the number of writes do not exceed the maximum. if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest { return nil, ErrTooManyWrites @@ -93,6 +101,29 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic opt = &ExecOptions{} } + // Translate query keys to ids, if necessary. + for i := range q.Calls { + if err := e.translateCall(index, idx, q.Calls[i]); err != nil { + return nil, err + } + } + + results, err := e.execute(ctx, index, q, slices, opt) + if err != nil { + return nil, err + } + + // Translate response objects from ids to keys, if necessary. + for i := range results { + results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) + if err != nil { + return nil, err + } + } + return results, nil +} + +func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { // Don't bother calculating slices for query types that don't require it. needsSlices := needsSlices(q.Calls) @@ -1559,6 +1590,78 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu } } +func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error { + // Translate column key. + if idx.Keys() { + if value := callArgString(c, "col"); value != "" { + ids, err := e.TranslateStore.TranslateColumnsToUint64(index, []string{value}) + if err != nil { + return err + } + c.Args["col"] = ids[0] + } + } + + // Translate row key, if field is specified & key exists. + if fieldName := callArgString(c, "field"); fieldName != "" { + field := idx.Field(fieldName) + if field.Keys() { + if value := callArgString(c, "row"); value != "" { + ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value}) + if err != nil { + return err + } + c.Args["row"] = ids[0] + } + } + } + + // Translate child calls. + for _, child := range c.Children { + if err := e.translateCall(index, idx, child); err != nil { + return err + } + } + + return nil +} + +func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}) (interface{}, error) { + switch result := result.(type) { + case *Row: + if idx.Keys() { + other := &Row{Attrs: result.Attrs} + for _, segment := range result.Segments() { + for _, col := range segment.Columns() { + key, err := e.TranslateStore.TranslateColumnToString(index, col) + if err != nil { + return nil, err + } + other.Keys = append(other.Keys, key) + } + } + return other, nil + } + + case []Pair: + if fieldName := callArgString(call, "field"); fieldName != "" { + field := idx.Field(fieldName) + if field.Keys() { + other := make([]Pair, len(result)) + for i := range result { + key, err := e.TranslateStore.TranslateRowToString(index, fieldName, result[i].ID) + if err != nil { + return nil, err + } + other[i] = Pair{Key: key, Count: result[i].Count} + } + return other, nil + } + } + } + return result, nil +} + // errSliceUnavailable is a marker error if no nodes are available. var errSliceUnavailable = errors.New("slice unavailable") @@ -1670,3 +1773,12 @@ func (vc *ValCount) Larger(other ValCount) ValCount { Count: vc.Count, } } + +func callArgString(call *pql.Call, key string) string { + value, ok := call.Args[key] + if !ok { + return "" + } + s, _ := value.(string) + return s +} diff --git a/executor_test.go b/executor_test.go index 3d0d9854e..164133ecb 100644 --- a/executor_test.go +++ b/executor_test.go @@ -22,6 +22,8 @@ import ( "testing" "github.com/davecgh/go-spew/spew" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" @@ -101,6 +103,35 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } }) + + t.Run("Keys", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) + if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } + + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + // Set bits. + if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ + `SetBit(field=f, row="bar", col="foo")`+"\n"+ + `SetBit(field=f, row="baz", col="foo")`+"\n"+ + `SetBit(field=f, row="bar", col="bat")`+"\n"+ + `SetBit(field=f, row="bbb", col="aaa")`+"\n", + ), nil, nil); err != nil { + t.Fatal(err) + } + + if results, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row="bar", field=f)`), nil, nil); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(results, []interface{}{ + &pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}}, + }, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" { + t.Fatal(diff) + } + }) } // Ensure a difference query can be executed. @@ -383,36 +414,36 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Ensure a TopN() query can be executed. func TestExecutor_Execute_TopN(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + t.Run("ID", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - // Set columns for rows 0, 10, & 20 across two slices. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(field=f, row=0, col=0) - SetBit(field=f, row=0, col=1) - SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(field=f, row=10, col=0) - SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=other, row=0, col=0) - `), nil, nil); err != nil { - t.Fatal(err) - } + // Set columns for rows 0, 10, & 20 across two slices. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(field=f, row=0, col=0) + SetBit(field=f, row=0, col=1) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(field=f, row=10, col=0) + SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=other, row=0, col=0) + `), nil, nil); err != nil { + t.Fatal(err) + } - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() - t.Run("Standard", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ @@ -422,6 +453,46 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) + + t.Run("Keys", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + // Set columns for rows 0, 10, & 20 across two slices. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(field=f, row="foo", col="a") + SetBit(field=f, row="foo", col="b") + SetBit(field=f, row="foo", col="c") + SetBit(field=f, row="foo", col="d") + SetBit(field=f, row="foo", col="e") + SetBit(field=f, row="bar", col="a") + SetBit(field=f, row="bar", col="b") + SetBit(field=f, row="baz", col="b") + SetBit(field=other, row="foo", col="a") + `), nil, nil); err != nil { + t.Fatal(err) + } + + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(result, []interface{}{ + []pilosa.Pair{ + {Key: "foo", Count: 5}, + {Key: "bar", Count: 2}, + }, + }); diff != "" { + t.Fatal(diff) + } + }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { @@ -1213,6 +1284,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() + hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.MaxWritesPerRequest = 3 if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites { diff --git a/field.go b/field.go index 2c7598037..d9c88c924 100644 --- a/field.go +++ b/field.go @@ -282,6 +282,7 @@ func (f *Field) loadMeta() error { f.options.Min = pb.Min f.options.Max = pb.Max f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum) + f.options.Keys = pb.Keys return nil } @@ -317,6 +318,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Min = 0 f.options.Max = 0 f.options.TimeQuantum = "" + f.options.Keys = opt.Keys case FieldTypeInt: f.options.Type = opt.Type f.options.CacheType = CacheTypeNone @@ -324,6 +326,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Min = opt.Min f.options.Max = opt.Max f.options.TimeQuantum = "" + f.options.Keys = opt.Keys // Create new bsiGroup. bsig := &bsiGroup{ @@ -345,6 +348,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.CacheSize = 0 f.options.Min = 0 f.options.Max = 0 + f.options.Keys = opt.Keys // Set the time quantum. if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { f.Close() @@ -378,6 +382,13 @@ func (f *Field) Close() error { return nil } +// Keys returns true if the field uses string keys. +func (f *Field) Keys() bool { + f.mu.RLock() + defer f.mu.RUnlock() + return f.options.Keys +} + // bsiGroup returns a bsiGroup by name. func (f *Field) bsiGroup(name string) *bsiGroup { f.mu.RLock() @@ -1038,6 +1049,7 @@ type FieldOptions struct { Min int64 `json:"min,omitempty"` Max int64 `json:"max,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + Keys bool `json:"keys,omitempty"` } // Validate ensures that FieldOption values are valid. @@ -1075,6 +1087,7 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { Min: o.Min, Max: o.Max, TimeQuantum: string(o.TimeQuantum), + Keys: o.Keys, } } @@ -1089,6 +1102,7 @@ func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions { Min: options.Min, Max: options.Max, TimeQuantum: TimeQuantum(options.TimeQuantum), + Keys: options.Keys, } } diff --git a/holder.go b/holder.go index 7cb285d78..316ca1244 100644 --- a/holder.go +++ b/holder.go @@ -111,7 +111,8 @@ func (h *Holder) Open() error { } for _, fi := range fis { - if !fi.IsDir() { + // Skip files or hidden directories. + if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") { continue } @@ -338,12 +339,15 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { return nil, errors.Wrap(err, "creating") } + index.keys = opt.Keys + if err := index.Open(); err != nil { return nil, errors.Wrap(err, "opening") + } else if err := index.saveMeta(); err != nil { + return nil, errors.Wrap(err, "meta") } // Update options. - h.indexes[index.Name()] = index return index, nil diff --git a/http/handler.go b/http/handler.go index 41a5faf55..bc09def3b 100644 --- a/http/handler.go +++ b/http/handler.go @@ -203,6 +203,8 @@ func NewRouter(handler *Handler) *mux.Router { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") + router.HandleFunc("/translate/data", handler.handleGetTranslateData).Methods("GET") + router.Use(handler.queryArgValidator) return router } @@ -1184,6 +1186,56 @@ func (h *Handler) GetAPI() *pilosa.API { type defaultClusterMessageResponse struct{} +// TranslateStoreBufferSize is the buffer size used for streaming data. +const TranslateStoreBufferSize = 65536 + +func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + offset, _ := strconv.ParseInt(q.Get("offset"), 10, 64) + + rc, err := h.API.TranslateStore.Reader(r.Context(), offset) + if err == pilosa.ErrNotImplemented { + http.Error(w, err.Error(), http.StatusNotImplemented) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer rc.Close() + + // Ensure reader is closed when the client disconnects. + go func() { <-r.Context().Done(); rc.Close() }() + + // Flush header so client can continue. + w.WriteHeader(http.StatusOK) + if w, ok := w.(http.Flusher); ok { + w.Flush() + } + + // Copy from reader to client until store or client disconnect. + buf := make([]byte, TranslateStoreBufferSize) + for { + // Read from store. + n, err := rc.Read(buf) + if err == io.EOF { + return + } else if err != nil { + h.Logger.Printf("http: translate store read error: %s", err) + return + } else if n == 0 { + continue + } + + // Write to response & flush. + if _, err := w.Write(buf[:n]); err != nil { + h.Logger.Printf("http: translate store response write error: %s", err) + return + } else if w, ok := w.(http.Flusher); ok { + w.Flush() + } + } +} + type queryValidationSpec struct { required []string args map[string]struct{} diff --git a/http/translator.go b/http/translator.go new file mode 100644 index 000000000..3ca9b840d --- /dev/null +++ b/http/translator.go @@ -0,0 +1,87 @@ +package http + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + + "github.com/pilosa/pilosa" +) + +// Ensure implementation implements inteface. +var _ pilosa.TranslateStore = (*TranslateStore)(nil) + +// TranslateStore represents an implementation of TranslateStore that +// communicates over HTTP. This is used with the TranslateHandler. +type TranslateStore struct { + URL string +} + +// NewTranslateStore returns a new instance of TranslateStore. +func NewTranslateStore(rawurl string) *TranslateStore { + return &TranslateStore{URL: rawurl} +} + +// TranslateColumnsToUint64 is not currently implemented. +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + return nil, pilosa.ErrNotImplemented +} + +// TranslateColumnToString is not currently implemented. +func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) { + return "", pilosa.ErrNotImplemented +} + +// TranslateRowsToUint64 is not currently implemented. +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + return nil, pilosa.ErrNotImplemented +} + +// TranslateRowToString is not currently implemented. +func (s *TranslateStore) TranslateRowToString(index, frame string, values uint64) (string, error) { + return "", pilosa.ErrNotImplemented +} + +// Reader returns a reader that can stream data from a remote store. +func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { + // Generate remote URL. + u, err := url.Parse(s.URL) + if err != nil { + return nil, err + } + u.Path = "/translate/data" + u.RawQuery = (url.Values{ + "offset": {strconv.FormatInt(off, 10)}, + }).Encode() + + // Connect a stream to the remote server. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + + // Connect a stream to the remote server. + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("http: cannot connect to translate store endpoint: %s", err) + } + + // Handle error codes or return body as stream. + switch resp.StatusCode { + case http.StatusOK: + return resp.Body, nil + case http.StatusNotImplemented: + resp.Body.Close() + return nil, pilosa.ErrNotImplemented + default: + body, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("http: invalid translate store endpoint status: code=%d url=%s body=%q", resp.StatusCode, u.String(), bytes.TrimSpace(body)) + } +} diff --git a/http/translator_test.go b/http/translator_test.go new file mode 100644 index 000000000..3378ddc58 --- /dev/null +++ b/http/translator_test.go @@ -0,0 +1,134 @@ +package http_test + +import ( + "context" + "io" + "io/ioutil" + "net/http/httptest" + "testing" + "time" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/mock" + "github.com/pilosa/pilosa/test" +) + +func TestTranslateStore_Reader(t *testing.T) { + // Ensure client can connect and stream the translate store data. + t.Run("OK", func(t *testing.T) { + t.Run("ServerDisconnect", func(t *testing.T) { + var mrc mock.ReadCloser + var readN int + mrc.ReadFunc = func(p []byte) (int, error) { + readN++ + switch readN { + case 1: + copy(p, []byte("foo")) + return 3, nil + case 2: + copy(p, []byte("barbaz")) + return 6, nil + case 3: + return 0, io.EOF + default: + t.Fatal("unexpected read") + return 0, nil + } + } + var closeInvoked bool + mrc.CloseFunc = func() error { + closeInvoked = true + return nil + } + + // Setup handler on test server. + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + if off != 100 { + t.Fatalf("unexpected off: %d", off) + } + return &mrc, nil + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + + // Connect to server and stream all available data. + store := http.NewTranslateStore(s.URL) + rc, err := store.Reader(context.Background(), 100) + if err != nil { + t.Fatal(err) + } else if data, err := ioutil.ReadAll(rc); err != nil { + t.Fatal(err) + } else if string(data) != `foobarbaz` { + t.Fatalf("unexpected data: %q", data) + } else if err := rc.Close(); err != nil { + t.Fatal(err) + } + + if !closeInvoked { + t.Fatal("expected server close") + } + }) + + // Ensure server closes store reader if client disconnects. + t.Run("ClientDisconnect", func(t *testing.T) { + // Setup mock so that Read() hangs. + done := make(chan struct{}) + + var mrc mock.ReadCloser + mrc.ReadFunc = func(p []byte) (int, error) { + <-done + return 0, io.EOF + } + var closeInvoked bool + mrc.CloseFunc = func() error { + closeInvoked = true + return nil + } + + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + return &mrc, nil + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + defer close(done) + + // Connect to server and begin streaming. + ctx, cancel := context.WithCancel(context.Background()) + store := http.NewTranslateStore(s.URL) + if _, err := store.Reader(ctx, 0); err != nil { + t.Fatal(err) + } + + // Cancel the context and check if server is closed. + cancel() + time.Sleep(100 * time.Millisecond) + if !closeInvoked { + t.Fatal("expected server-side close") + } + }) + }) + + // Ensure client is notified if the server doesn't support streaming replication. + t.Run("ErrNotImplemented", func(t *testing.T) { + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + return nil, pilosa.ErrNotImplemented + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + + _, err := http.NewTranslateStore(s.URL).Reader(context.Background(), 0) + if err != pilosa.ErrNotImplemented { + t.Fatalf("unexpected error: %s", err) + } + }) +} diff --git a/index.go b/index.go index 46ca48e47..68c48c829 100644 --- a/index.go +++ b/index.go @@ -33,6 +33,7 @@ type Index struct { mu sync.RWMutex path string name string + keys bool // use string keys // Fields by name. fields map[string]*Field @@ -80,6 +81,9 @@ func (i *Index) Name() string { return i.name } // Path returns the path the index was initialized with. func (i *Index) Path() string { return i.path } +// Keys returns true if the index uses string keys. +func (i *Index) Keys() bool { return i.keys } + // ColumnAttrStore returns the storage for column attributes. func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore } @@ -164,18 +168,17 @@ func (i *Index) loadMeta() error { } // Copy metadata fields. + i.keys = pb.Keys return nil } -// NOTE: Until we introduce new attributes to store in the index .meta file, -// we don't need to actually write the file. The code related to index.options -// and the index meta file are left in place for future use. -/* // saveMeta writes meta data for the index. func (i *Index) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.IndexMeta{}) + buf, err := proto.Marshal(&internal.IndexMeta{ + Keys: i.keys, + }) if err != nil { return errors.Wrap(err, "marshalling") } @@ -187,7 +190,6 @@ func (i *Index) saveMeta() error { return nil } -*/ // Close closes the index and its fields. func (i *Index) Close() error { @@ -407,11 +409,15 @@ func encodeIndex(d *Index) *internal.Index { } // IndexOptions represents options to set when initializing an index. -type IndexOptions struct{} +type IndexOptions struct { + Keys bool `json:"keys"` +} // Encode converts i into its internal representation. func (i *IndexOptions) Encode() *internal.IndexMeta { - return &internal.IndexMeta{} + return &internal.IndexMeta{ + Keys: i.Keys, + } } // hasTime returns true if a contains a non-nil time. diff --git a/inmem/translator.go b/inmem/translator.go new file mode 100644 index 000000000..b620b2d03 --- /dev/null +++ b/inmem/translator.go @@ -0,0 +1,215 @@ +package inmem + +import ( + "context" + "io" + "sync" + + "github.com/pilosa/pilosa" +) + +// Ensure type implements interface. +var _ pilosa.TranslateStore = &TranslateStore{} + +// TranslateStore is an in-memory storage engine for translating string-to-uint64 values. +type TranslateStore struct { + mu sync.RWMutex + + cols map[string]*translateIndex + rows map[frameKey]*translateIndex +} + +// NewTranslateStore returns a new instance of TranslateStore. +func NewTranslateStore() *TranslateStore { + return &TranslateStore{ + cols: make(map[string]*translateIndex), + rows: make(map[frameKey]*translateIndex), + } +} + +// Reader returns an error because it is not supported by the inmem store. +func (s *TranslateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { + return nil, pilosa.ErrReplicationNotSupported +} + +// TranslateColumnsToUint64 converts value to a uint64 id. +// If value does not have an associated id then one is created. +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.cols[index] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create index map if it doesn't exists. + if idx == nil { + idx = newTranslateIndex() + s.cols[index] = idx + } + + // Add new identifiers. + for i := range values { + if ret[i] != 0 { + continue + } + + idx.seq++ + v := idx.seq + ret[i] = v + idx.lookup[values[i]] = v + idx.reverse[v] = values[i] + } + + return ret, nil +} + +// TranslateColumnToString converts a uint64 id to its associated string value. +// If the id is not associated with a string value then a blank string is returned. +func (s *TranslateStore) TranslateColumnToString(index string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + if ret, ok := idx.reverse[value]; ok { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + return "", nil +} + +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + key := frameKey{index, frame} + + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.rows[key]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.rows[key] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create map if it doesn't exists. + if idx == nil { + idx = newTranslateIndex() + s.rows[key] = idx + } + + // Add new identifiers. + for i := range values { + if ret[i] != 0 { + continue + } + + idx.seq++ + v := idx.seq + ret[i] = v + idx.lookup[values[i]] = v + idx.reverse[v] = values[i] + } + + return ret, nil +} + +func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.rows[frameKey{index, frame}]; idx != nil { + if ret, ok := idx.reverse[value]; ok { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + return "", nil +} + +type frameKey struct { + index string + frame string +} + +type translateIndex struct { + seq uint64 + lookup map[string]uint64 + reverse map[uint64]string +} + +func newTranslateIndex() *translateIndex { + return &translateIndex{ + lookup: make(map[string]uint64), + reverse: make(map[uint64]string), + } +} diff --git a/inmem/translator_test.go b/inmem/translator_test.go new file mode 100644 index 000000000..d4d232566 --- /dev/null +++ b/inmem/translator_test.go @@ -0,0 +1,132 @@ +package inmem_test + +import ( + "fmt" + "math/rand" + "reflect" + "testing" + + "github.com/pilosa/pilosa/inmem" +) + +func TestTranslateStore_TranslateColumn(t *testing.T) { + s := inmem.NewTranslateStore() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } +} + +func TestTranslateStore_TranslateRow(t *testing.T) { + s := inmem.NewTranslateStore() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } +} + +func BenchmarkTranslateStore_TranslateColumnsToUint64(b *testing.B) { + const batchSize = 1000 + + s := inmem.NewTranslateStore() + + // Generate keys before benchmark begins + keySets := make([][]string, b.N/1000) + for i := range keySets { + keySets[i] = make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i) + } + } + + b.ResetTimer() + + for _, keySet := range keySets { + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkTranslateStore_TranslateColumnToString(b *testing.B) { + const batchSize = 1000 + + s := inmem.NewTranslateStore() + + // Generate keys before benchmark begins + for i := 0; i < b.N; i += batchSize { + keySet := make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySet[j] = fmt.Sprintf("%08d%08d", jv, i) + } + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } + + // Generate random key access. + perm := rand.New(rand.NewSource(0)).Perm(b.N) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/private.pb.go b/internal/private.pb.go index 1e2b34f38..c3dadb455 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: private.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -61,6 +60,7 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { + Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` } func (m *IndexMeta) Reset() { *m = IndexMeta{} } @@ -68,6 +68,13 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } +func (m *IndexMeta) GetKeys() bool { + if m != nil { + return m.Keys + } + return false +} + type FieldOptions struct { Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` @@ -75,6 +82,7 @@ type FieldOptions struct { Min int64 `protobuf:"varint,9,opt,name=Min,proto3" json:"Min,omitempty"` Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"` TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"` } func (m *FieldOptions) Reset() { *m = FieldOptions{} } @@ -124,6 +132,13 @@ func (m *FieldOptions) GetTimeQuantum() string { return "" } +func (m *FieldOptions) GetKeys() bool { + if m != nil { + return m.Keys + } + return false +} + type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` } @@ -966,6 +981,16 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Keys { + dAtA[i] = 0x18 + i++ + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -1017,6 +1042,16 @@ func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) } + if m.Keys { + dAtA[i] = 0x58 + i++ + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -2105,24 +2140,6 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Private(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2135,6 +2152,9 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *IndexMeta) Size() (n int) { var l int _ = l + if m.Keys { + n += 2 + } return n } @@ -2162,6 +2182,9 @@ func (m *FieldOptions) Size() (n int) { if m.Max != 0 { n += 1 + sovPrivate(uint64(m.Max)) } + if m.Keys { + n += 2 + } return n } @@ -2675,6 +2698,26 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Keys = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -2869,6 +2912,26 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { break } } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Keys = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3485,51 +3548,14 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var keykey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - keykey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey if m.Standard == nil { m.Standard = make(map[string]uint64) } - if iNdEx < postIndex { - var valuekey uint64 + var mapkey string + var mapvalue uint64 + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3539,31 +3565,69 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - valuekey |= (uint64(b) & 0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - var mapvalue uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy } - m.Standard[mapkey] = mapvalue - } else { - var mapvalue uint64 - m.Standard[mapkey] = mapvalue } + m.Standard[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -6617,69 +6681,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1011 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1c, 0x35, - 0x18, 0x67, 0x1e, 0xbb, 0xd9, 0xfd, 0xd2, 0x0d, 0x89, 0x0b, 0x61, 0x8a, 0x50, 0x58, 0xac, 0x4a, - 0x0d, 0x3d, 0x44, 0xa5, 0xbd, 0xf0, 0xaa, 0x14, 0x25, 0x1b, 0x60, 0x10, 0x09, 0xe0, 0x49, 0x7a, - 0xeb, 0xc1, 0xdd, 0xb5, 0xda, 0x51, 0x66, 0xc7, 0xc3, 0x8c, 0x27, 0xc9, 0xf6, 0xc0, 0x15, 0x2e, - 0xdc, 0x11, 0x67, 0xfe, 0x18, 0x8e, 0xfc, 0x09, 0x28, 0xfc, 0x23, 0xc8, 0x9f, 0x3d, 0x8f, 0x64, - 0x37, 0x4d, 0x15, 0x7a, 0xf3, 0xf7, 0x7e, 0xfd, 0x3e, 0xdb, 0x30, 0xc8, 0xf2, 0xf8, 0x84, 0x2b, - 0xb1, 0x95, 0xe5, 0x52, 0x49, 0xd2, 0x8b, 0x53, 0x25, 0xf2, 0x94, 0x27, 0x74, 0x19, 0xfa, 0x61, - 0x3a, 0x11, 0x67, 0xfb, 0x42, 0x71, 0xfa, 0xa7, 0x03, 0xb7, 0xbe, 0x8a, 0x45, 0x32, 0xf9, 0x3e, - 0x53, 0xb1, 0x4c, 0x0b, 0xf2, 0x01, 0xf4, 0x77, 0xf9, 0xf8, 0x85, 0x38, 0x9c, 0x65, 0x22, 0xf0, - 0x86, 0xce, 0x66, 0x9f, 0x35, 0x8c, 0x5a, 0x1a, 0xc5, 0x2f, 0x45, 0xe0, 0x0f, 0x9d, 0xcd, 0x01, - 0x6b, 0x18, 0x64, 0x08, 0xcb, 0x87, 0xf1, 0x54, 0xfc, 0x58, 0xf2, 0x54, 0x95, 0xd3, 0xa0, 0x83, - 0xd6, 0x6d, 0x16, 0x21, 0xe0, 0xa3, 0xe3, 0x1e, 0x8a, 0xf0, 0x4c, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, - 0xa0, 0x3f, 0x74, 0x36, 0x3d, 0xa6, 0x8f, 0xc8, 0xe1, 0x67, 0x01, 0x58, 0x0e, 0x3f, 0xa3, 0x14, - 0x56, 0xc2, 0x69, 0x26, 0x73, 0xc5, 0x44, 0x91, 0xc9, 0xb4, 0x40, 0xab, 0xbd, 0x3c, 0x0f, 0x1c, - 0x74, 0xa4, 0x8f, 0xf4, 0x67, 0x58, 0xdd, 0x49, 0xe4, 0xf8, 0x78, 0xc4, 0x15, 0x67, 0xe2, 0xa7, - 0x52, 0x14, 0x8a, 0xbc, 0x03, 0x1d, 0xac, 0xd5, 0xea, 0x19, 0x42, 0x73, 0xb1, 0xe6, 0xc0, 0x35, - 0x5c, 0x24, 0x34, 0x17, 0xed, 0xb1, 0x6a, 0x9f, 0x19, 0x42, 0x73, 0xa3, 0x24, 0x1e, 0x9b, 0x6a, - 0x7d, 0x66, 0x08, 0x5d, 0xc7, 0x93, 0x58, 0x9c, 0xda, 0x12, 0xf1, 0x4c, 0x43, 0x58, 0x6b, 0xc5, - 0xb7, 0x69, 0xae, 0x43, 0x97, 0xc9, 0xd3, 0x70, 0x54, 0x04, 0xce, 0xd0, 0xdb, 0xf4, 0x99, 0xa5, - 0xb0, 0x91, 0x32, 0x29, 0xa7, 0xa9, 0x16, 0xb9, 0x28, 0x6a, 0x18, 0xf4, 0x0e, 0x74, 0xb0, 0xab, - 0xba, 0xca, 0xc6, 0x56, 0x1f, 0xe9, 0x2f, 0x0e, 0xf4, 0xf7, 0xf9, 0x19, 0xa6, 0x51, 0x90, 0xc7, - 0xd0, 0x8b, 0x14, 0x4f, 0x27, 0x3c, 0x9f, 0xa0, 0xd2, 0xf2, 0xc3, 0x8f, 0xb6, 0xaa, 0x41, 0x6f, - 0xd5, 0x6a, 0x5b, 0x95, 0xce, 0x5e, 0xaa, 0xf2, 0x19, 0xab, 0x4d, 0xde, 0xff, 0x02, 0x06, 0x17, - 0x44, 0x3a, 0xde, 0xb1, 0x98, 0x55, 0x5d, 0x3d, 0x16, 0x33, 0x5d, 0xff, 0x09, 0x4f, 0x4a, 0x81, - 0xbd, 0xf2, 0x99, 0x21, 0x3e, 0x77, 0x3f, 0x75, 0xe8, 0x36, 0x90, 0xdd, 0x5c, 0x70, 0x25, 0x30, - 0xc8, 0xbe, 0x28, 0x0a, 0xfe, 0x5c, 0x5c, 0xdd, 0x71, 0xd3, 0x45, 0xb7, 0xd5, 0x45, 0x7a, 0x1f, - 0xc8, 0x48, 0x24, 0x42, 0x09, 0x8b, 0xc7, 0x57, 0x78, 0xa0, 0x51, 0x15, 0xed, 0x7a, 0x5d, 0x72, - 0x0f, 0x7c, 0x0d, 0x6e, 0x0c, 0xb6, 0xfc, 0xf0, 0x76, 0xd3, 0x91, 0x1a, 0xf7, 0x0c, 0x15, 0x68, - 0x52, 0x39, 0x45, 0x04, 0x5c, 0x5b, 0xc2, 0x02, 0xd0, 0xdc, 0xb7, 0xa1, 0x3c, 0x0c, 0xb5, 0xde, - 0x84, 0x6a, 0x2f, 0x95, 0x8d, 0xb6, 0x5d, 0x95, 0x7b, 0xd3, 0x68, 0xf4, 0xa9, 0xe5, 0x6a, 0xfc, - 0x1d, 0xf0, 0xa9, 0xb0, 0x36, 0x78, 0xae, 0x53, 0x71, 0xaf, 0x4f, 0x45, 0xbb, 0xd7, 0x98, 0x2d, - 0x02, 0x6f, 0xe8, 0x69, 0xf7, 0x48, 0xd0, 0x47, 0xd0, 0x8d, 0xc6, 0x2f, 0xc4, 0x94, 0x93, 0x8f, - 0x61, 0x09, 0xf3, 0x10, 0x85, 0x85, 0xd5, 0xdb, 0x97, 0x9a, 0xc8, 0x2a, 0x39, 0x1d, 0xd9, 0xfc, - 0x17, 0xe6, 0x74, 0x0f, 0xba, 0x18, 0xbd, 0x08, 0xfc, 0xcb, 0x6e, 0x90, 0xcf, 0xac, 0x98, 0xee, - 0x81, 0x77, 0xc4, 0x42, 0xbd, 0x2e, 0x98, 0x41, 0xe5, 0xc5, 0x52, 0xda, 0xf7, 0x37, 0xb2, 0x50, - 0xb6, 0x1b, 0x78, 0xd6, 0xbc, 0x1f, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x67, 0xfa, 0x14, 0xfc, - 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x7d, 0xb8, 0xe1, 0x88, 0x7c, 0x88, 0xee, 0x6d, - 0x6b, 0x06, 0x4d, 0x12, 0x47, 0x2c, 0x64, 0x18, 0xf8, 0x2e, 0x0c, 0xc2, 0x62, 0x57, 0xca, 0x7c, - 0x12, 0xa7, 0x5c, 0xc9, 0x1c, 0xbd, 0xf6, 0xd8, 0x45, 0x26, 0xdd, 0x86, 0x55, 0xed, 0x3e, 0x52, - 0x5c, 0xd5, 0x80, 0x5f, 0x87, 0xae, 0xe6, 0xd5, 0xe1, 0x2c, 0x85, 0x90, 0xd7, 0x7a, 0xd5, 0x04, - 0x91, 0xa0, 0xdf, 0x19, 0x0f, 0x7b, 0x27, 0x22, 0x55, 0x2d, 0x04, 0x20, 0x8d, 0x0e, 0x06, 0xcc, - 0x10, 0x84, 0x9a, 0x52, 0x6c, 0xce, 0x2b, 0x4d, 0xce, 0x9a, 0xcb, 0x50, 0x46, 0x7f, 0x73, 0x00, - 0xaa, 0x84, 0xca, 0xa2, 0x36, 0x71, 0xae, 0x36, 0x21, 0x9f, 0xb4, 0xae, 0x8f, 0xf9, 0x05, 0xa9, - 0x45, 0xac, 0x75, 0xc9, 0x6c, 0x56, 0xb0, 0xb0, 0x28, 0x5f, 0x6d, 0xf4, 0x0d, 0xdf, 0x8e, 0x89, - 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0xdb, 0x8c, 0xf4, 0x35, 0x67, 0x18, 0x75, 0x7f, - 0x1a, 0xc6, 0xe2, 0x16, 0x91, 0xbb, 0xd0, 0xd1, 0x99, 0x1a, 0x6c, 0xce, 0x97, 0x61, 0x84, 0xf4, - 0x09, 0xf4, 0x76, 0xa2, 0xf0, 0xeb, 0x5c, 0x96, 0xd9, 0x42, 0xe4, 0x55, 0x2f, 0x8d, 0x3b, 0xff, - 0xd2, 0x78, 0x73, 0x2f, 0x8d, 0xdf, 0xbc, 0x34, 0x11, 0xac, 0x99, 0x2b, 0x41, 0xaf, 0xc4, 0x4d, - 0x6e, 0x84, 0xea, 0x69, 0xf0, 0x5a, 0x4f, 0x43, 0x04, 0x6b, 0x66, 0xf3, 0xdf, 0xa4, 0xd3, 0x3f, - 0x5c, 0x58, 0x63, 0xa2, 0x88, 0x5f, 0x8a, 0x30, 0x2d, 0x54, 0x5e, 0x8e, 0xf5, 0x82, 0x6b, 0xfb, - 0x6f, 0xe5, 0x33, 0xdb, 0x6d, 0x8f, 0x19, 0xe2, 0x75, 0xc0, 0x44, 0x1e, 0xc0, 0xf2, 0xe5, 0x05, - 0x98, 0x57, 0x6d, 0xab, 0x90, 0x07, 0xb0, 0x14, 0xc9, 0x32, 0xd7, 0x48, 0x32, 0xeb, 0xdd, 0xba, - 0x74, 0x4c, 0x66, 0x46, 0xcc, 0x2a, 0xb5, 0x16, 0x94, 0x3a, 0xaf, 0x86, 0x12, 0x79, 0x7c, 0x09, - 0x4a, 0x41, 0x17, 0x0d, 0xde, 0x6b, 0x0c, 0x2e, 0x88, 0xd9, 0x45, 0x6d, 0xfa, 0xab, 0x03, 0xb7, - 0xda, 0x29, 0xbc, 0xd6, 0x6e, 0xd4, 0x13, 0x71, 0x17, 0x4e, 0xc4, 0x5b, 0x34, 0x11, 0xbf, 0x99, - 0x48, 0xf3, 0xca, 0x75, 0xda, 0xaf, 0xdc, 0x31, 0xdc, 0x99, 0x1b, 0xd3, 0xae, 0x9c, 0x66, 0x1a, - 0x0f, 0xff, 0x63, 0x5c, 0xfa, 0xd6, 0xc8, 0x73, 0x3b, 0xa8, 0x3e, 0x33, 0x04, 0xfd, 0x0c, 0xde, - 0x8d, 0x84, 0x6a, 0x0d, 0xa9, 0x42, 0xdb, 0x10, 0xbc, 0x03, 0x71, 0x7a, 0x45, 0xf9, 0x5a, 0x44, - 0xbf, 0x84, 0xe0, 0x28, 0x9b, 0x70, 0x25, 0x6e, 0x64, 0xbd, 0x03, 0xbd, 0x43, 0x99, 0xc9, 0x44, - 0x3e, 0x9f, 0x5d, 0xb3, 0xf5, 0x01, 0x2c, 0x99, 0x2b, 0xd2, 0x7c, 0x7c, 0xfa, 0xac, 0x22, 0xe9, - 0x6d, 0x0d, 0xe8, 0x31, 0x4f, 0xc6, 0x65, 0xa2, 0xd3, 0xd0, 0x3f, 0xa0, 0x62, 0x67, 0xf5, 0xaf, - 0xf3, 0x0d, 0xe7, 0xef, 0xf3, 0x0d, 0xe7, 0x9f, 0xf3, 0x0d, 0xe7, 0xf7, 0x7f, 0x37, 0xde, 0x7a, - 0xd6, 0xc5, 0x1f, 0xed, 0xa3, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x97, 0xf0, 0x12, 0xfd, 0xe2, - 0x0a, 0x00, 0x00, + // 1028 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x72, 0x1c, 0x35, + 0x17, 0xfe, 0xfb, 0x32, 0xe3, 0x99, 0xe3, 0x8c, 0x7f, 0x5b, 0x01, 0xd3, 0xa1, 0x28, 0x67, 0x50, + 0xa5, 0x2a, 0x26, 0x0b, 0x57, 0x48, 0x36, 0xdc, 0x52, 0xe5, 0xb2, 0xc7, 0x40, 0x03, 0x36, 0xa0, + 0xb6, 0xb3, 0xcb, 0x42, 0x99, 0x51, 0x25, 0x5d, 0xee, 0x69, 0x35, 0xdd, 0x6a, 0xdb, 0x93, 0x05, + 0x5b, 0xd8, 0xb0, 0xa7, 0x78, 0x12, 0x1e, 0x81, 0x25, 0x8f, 0x40, 0x99, 0x17, 0xa1, 0x74, 0xa4, + 0xbe, 0xd8, 0x33, 0x8e, 0x53, 0x86, 0x9d, 0xce, 0xfd, 0xd3, 0xd1, 0x77, 0x24, 0xc1, 0x20, 0xcb, + 0xe3, 0x13, 0xae, 0xc4, 0x56, 0x96, 0x4b, 0x25, 0x49, 0x2f, 0x4e, 0x95, 0xc8, 0x53, 0x9e, 0xd0, + 0xbb, 0xd0, 0x0f, 0xd3, 0x89, 0x38, 0xdb, 0x17, 0x8a, 0x13, 0x02, 0xfe, 0xd7, 0x62, 0x56, 0x04, + 0xde, 0xd0, 0xd9, 0xec, 0x31, 0x5c, 0xd3, 0xdf, 0x1d, 0xb8, 0xf5, 0x79, 0x2c, 0x92, 0xc9, 0xb7, + 0x99, 0x8a, 0x65, 0x5a, 0x90, 0xf7, 0xa0, 0xbf, 0xcb, 0xc7, 0x2f, 0xc5, 0xe1, 0x2c, 0x13, 0xe8, + 0xd9, 0x67, 0x8d, 0xa2, 0xb6, 0x46, 0xf1, 0x2b, 0x11, 0xf8, 0x43, 0x67, 0x73, 0xc0, 0x1a, 0x05, + 0x19, 0xc2, 0xf2, 0x61, 0x3c, 0x15, 0xdf, 0x97, 0x3c, 0x55, 0xe5, 0x34, 0xe8, 0x60, 0x74, 0x5b, + 0xa5, 0x21, 0x60, 0xe2, 0x1e, 0x9a, 0x70, 0x4d, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, 0xa0, 0x3f, 0x74, + 0x36, 0x3d, 0xa6, 0x97, 0xa8, 0xe1, 0x67, 0x01, 0x58, 0x0d, 0x3f, 0xab, 0xa1, 0x2f, 0xb7, 0xa0, + 0x53, 0x58, 0x09, 0xa7, 0x99, 0xcc, 0x15, 0x13, 0x45, 0x26, 0xd3, 0x02, 0x33, 0xed, 0xe5, 0x79, + 0xe0, 0x60, 0x72, 0xbd, 0xa4, 0x3f, 0xc2, 0xea, 0x4e, 0x22, 0xc7, 0xc7, 0x23, 0xae, 0x38, 0x13, + 0x3f, 0x94, 0xa2, 0x50, 0xe4, 0x2d, 0xe8, 0x60, 0x4f, 0xac, 0x9f, 0x11, 0xb4, 0x16, 0xfb, 0x10, + 0xb8, 0x46, 0x8b, 0x82, 0xd6, 0x62, 0x3c, 0x76, 0xc2, 0x67, 0x46, 0xd0, 0xda, 0x28, 0x89, 0xc7, + 0xa6, 0x03, 0x3e, 0x33, 0x82, 0xc6, 0xf8, 0x34, 0x16, 0xa7, 0x76, 0xdb, 0xb8, 0xa6, 0x21, 0xac, + 0xb5, 0xea, 0x5b, 0x98, 0xeb, 0xd0, 0x65, 0xf2, 0x34, 0x1c, 0x15, 0x81, 0x33, 0xf4, 0x36, 0x7d, + 0x66, 0x25, 0x6c, 0xae, 0x4c, 0xca, 0x69, 0xaa, 0x4d, 0x2e, 0x9a, 0x1a, 0x05, 0xbd, 0x03, 0x1d, + 0xec, 0xb4, 0xde, 0x65, 0x13, 0xab, 0x97, 0xf4, 0x27, 0x07, 0xfa, 0xfb, 0xfc, 0x0c, 0x61, 0x14, + 0xe4, 0x09, 0xf4, 0x22, 0xc5, 0xd3, 0x09, 0xcf, 0x27, 0xe8, 0xb4, 0xfc, 0xe8, 0xfd, 0xad, 0x8a, + 0x10, 0x5b, 0xb5, 0xdb, 0x56, 0xe5, 0xb3, 0x97, 0xaa, 0x7c, 0xc6, 0xea, 0x90, 0x77, 0x3f, 0x85, + 0xc1, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x55, 0x57, 0x8f, 0xc5, 0x4c, 0xef, 0xff, 0x84, 0x27, + 0xa5, 0xc0, 0x5e, 0xf9, 0xcc, 0x08, 0x9f, 0xb8, 0x1f, 0x39, 0x74, 0x1b, 0xc8, 0x6e, 0x2e, 0xb8, + 0x12, 0x58, 0x64, 0x5f, 0x14, 0x05, 0x7f, 0x21, 0xae, 0xee, 0xb8, 0xe9, 0xa2, 0xdb, 0xea, 0x22, + 0x7d, 0x00, 0x64, 0x24, 0x12, 0xa1, 0x84, 0xe5, 0xed, 0x6b, 0x32, 0xd0, 0xa8, 0xaa, 0x76, 0xbd, + 0x2f, 0xb9, 0x0f, 0xbe, 0x1e, 0x02, 0x2c, 0xb6, 0xfc, 0xe8, 0x76, 0xd3, 0x91, 0x7a, 0x3e, 0x18, + 0x3a, 0xd0, 0xa4, 0x4a, 0x8a, 0x0c, 0xb8, 0x76, 0x0b, 0x0b, 0x48, 0xf3, 0xc0, 0x96, 0xf2, 0xb0, + 0xd4, 0x7a, 0x53, 0xaa, 0x3d, 0x68, 0xb6, 0xda, 0x76, 0xb5, 0xdd, 0x9b, 0x56, 0xa3, 0xcf, 0xac, + 0x56, 0xf3, 0xef, 0x80, 0x4f, 0x85, 0x8d, 0xc1, 0x75, 0x0d, 0xc5, 0xbd, 0x1e, 0x8a, 0x4e, 0xaf, + 0x39, 0xab, 0xef, 0x07, 0x4f, 0xa7, 0x47, 0x81, 0x3e, 0x86, 0x6e, 0x34, 0x7e, 0x29, 0xa6, 0x9c, + 0x7c, 0x00, 0x4b, 0x88, 0x43, 0x14, 0x96, 0x56, 0xff, 0xbf, 0xd4, 0x44, 0x56, 0xd9, 0xe9, 0xc8, + 0xe2, 0x5f, 0x88, 0xe9, 0x3e, 0x74, 0xb1, 0x7a, 0x11, 0xf8, 0x97, 0xd3, 0xa0, 0x9e, 0x59, 0x33, + 0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5c, 0x10, 0x41, 0x95, 0xc5, 0x4a, 0x3a, 0xf7, 0x97, 0xb2, + 0x50, 0xb6, 0x1b, 0xb8, 0xd6, 0xba, 0xef, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x6b, 0xfa, 0x0c, + 0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x73, 0xb8, 0xe1, 0x88, 0xdc, 0xc5, 0xf4, + 0xb6, 0x35, 0x83, 0x06, 0xc4, 0x11, 0x0b, 0x19, 0x16, 0xbe, 0x07, 0x83, 0xb0, 0xd8, 0x95, 0x32, + 0x9f, 0xc4, 0x29, 0x57, 0x32, 0xb7, 0x17, 0xe7, 0x45, 0x25, 0xdd, 0x86, 0x55, 0x9d, 0x3e, 0x52, + 0x5c, 0xd5, 0x84, 0x5f, 0x87, 0xae, 0xd6, 0xd5, 0xe5, 0xac, 0x84, 0x94, 0xd7, 0x7e, 0xd5, 0x09, + 0xa2, 0x40, 0xbf, 0x31, 0x19, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x0c, 0x40, 0x19, 0x13, 0x0c, 0x98, + 0x11, 0x08, 0x35, 0x5b, 0xb1, 0x98, 0x57, 0x1a, 0xcc, 0x5a, 0xcb, 0xd0, 0x46, 0x7f, 0x71, 0x00, + 0x2a, 0x40, 0x65, 0x51, 0x87, 0x38, 0x57, 0x87, 0x90, 0x0f, 0x5b, 0xd7, 0xc7, 0xfc, 0x80, 0xd4, + 0x26, 0xd6, 0xba, 0x64, 0x36, 0x2b, 0x5a, 0x58, 0x96, 0xaf, 0x36, 0xfe, 0x46, 0x6f, 0x8f, 0x89, + 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0x5b, 0x44, 0xfa, 0x9a, 0x33, 0x8a, 0xba, 0x3f, + 0x8d, 0x62, 0x71, 0x8b, 0xc8, 0x3d, 0xe8, 0x68, 0xa4, 0x86, 0x9b, 0xf3, 0xdb, 0x30, 0x46, 0xfa, + 0x14, 0x7a, 0x3b, 0x51, 0xf8, 0x45, 0x2e, 0xcb, 0x6c, 0x21, 0xf3, 0xaa, 0xd7, 0xc7, 0x9d, 0x7f, + 0x7d, 0xbc, 0xb9, 0xd7, 0xc7, 0xaf, 0x5f, 0x1f, 0x1a, 0xc1, 0x9a, 0xb9, 0x12, 0xf4, 0x48, 0xdc, + 0xe4, 0x46, 0xa8, 0x9e, 0x06, 0xaf, 0xf5, 0x34, 0x44, 0xb0, 0x66, 0x26, 0xff, 0xbf, 0x4c, 0xfa, + 0x9b, 0x0b, 0x6b, 0x4c, 0x14, 0xf1, 0x2b, 0x11, 0xa6, 0x85, 0xca, 0xcb, 0xb1, 0x1e, 0x70, 0x1d, + 0xff, 0x95, 0x7c, 0x6e, 0xbb, 0xed, 0x31, 0x23, 0xbc, 0x09, 0x99, 0xc8, 0x43, 0x58, 0xbe, 0x3c, + 0x00, 0xf3, 0xae, 0x6d, 0x17, 0xf2, 0x10, 0x96, 0x22, 0x59, 0xe6, 0x9a, 0x49, 0x66, 0xbc, 0x5b, + 0x97, 0x8e, 0x41, 0x66, 0xcc, 0xac, 0x72, 0x6b, 0x51, 0xa9, 0xf3, 0x7a, 0x2a, 0x91, 0x27, 0x97, + 0xa8, 0x14, 0x74, 0x31, 0xe0, 0x9d, 0x26, 0xe0, 0x82, 0x99, 0x5d, 0xf4, 0xa6, 0x3f, 0x3b, 0x70, + 0xab, 0x0d, 0xe1, 0x8d, 0x66, 0xa3, 0x3e, 0x11, 0x77, 0xe1, 0x89, 0x78, 0x8b, 0x4e, 0xc4, 0x6f, + 0x4e, 0xa4, 0x79, 0xe5, 0x3a, 0xed, 0x57, 0xee, 0x18, 0xee, 0xcc, 0x1d, 0xd3, 0xae, 0x9c, 0x66, + 0x9a, 0x0f, 0xff, 0xe2, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0xf6, 0xa0, 0xfa, 0xcc, 0x08, 0xf4, 0x63, + 0x78, 0x3b, 0x12, 0xaa, 0x75, 0x48, 0x15, 0xdb, 0x86, 0xe0, 0x1d, 0x88, 0xd3, 0x2b, 0xb6, 0xaf, + 0x4d, 0xf4, 0x33, 0x08, 0x8e, 0xb2, 0x09, 0x57, 0xe2, 0x46, 0xd1, 0x3b, 0xd0, 0x3b, 0x94, 0x99, + 0x4c, 0xe4, 0x8b, 0xd9, 0x35, 0x53, 0x1f, 0xc0, 0x92, 0xb9, 0x22, 0xcd, 0xc7, 0xa7, 0xcf, 0x2a, + 0x91, 0xde, 0xd6, 0x84, 0x1e, 0xf3, 0x64, 0x5c, 0x26, 0x1a, 0x86, 0xfe, 0x01, 0x15, 0x3b, 0xab, + 0x7f, 0x9c, 0x6f, 0x38, 0x7f, 0x9e, 0x6f, 0x38, 0x7f, 0x9d, 0x6f, 0x38, 0xbf, 0xfe, 0xbd, 0xf1, + 0xbf, 0xe7, 0x5d, 0xfc, 0xf9, 0x3e, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x25, 0x40, 0x21, + 0x0a, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 9cab31828..23bb4886c 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package internal; message IndexMeta { + bool Keys = 3; } message FieldOptions { @@ -12,6 +13,7 @@ message FieldOptions { int64 Min = 9; int64 Max = 10; string TimeQuantum = 5; + bool Keys = 11; } message ImportResponse { diff --git a/internal/public.pb.go b/internal/public.pb.go index 76266d2db..3cb6fa270 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: public.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -28,6 +27,8 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import encoding_binary "encoding/binary" + import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -799,7 +800,8 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue)))) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + i += 8 } return i, nil } @@ -1235,24 +1237,6 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Public(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Public(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2333,15 +2317,8 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } + v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 - v = uint64(dAtA[iNdEx-8]) - v |= uint64(dAtA[iNdEx-7]) << 8 - v |= uint64(dAtA[iNdEx-6]) << 16 - v |= uint64(dAtA[iNdEx-5]) << 24 - v |= uint64(dAtA[iNdEx-4]) << 32 - v |= uint64(dAtA[iNdEx-3]) << 40 - v |= uint64(dAtA[iNdEx-2]) << 48 - v |= uint64(dAtA[iNdEx-1]) << 56 m.FloatValue = float64(math.Float64frombits(v)) default: iNdEx = preIndex diff --git a/mock/mock.go b/mock/mock.go new file mode 100644 index 000000000..46469c2f9 --- /dev/null +++ b/mock/mock.go @@ -0,0 +1,14 @@ +package mock + +type ReadCloser struct { + ReadFunc func(p []byte) (int, error) + CloseFunc func() error +} + +func (rc *ReadCloser) Read(p []byte) (int, error) { + return rc.ReadFunc(p) +} + +func (rc *ReadCloser) Close() error { + return rc.CloseFunc() +} diff --git a/mock/translator.go b/mock/translator.go new file mode 100644 index 000000000..3f815b89f --- /dev/null +++ b/mock/translator.go @@ -0,0 +1,38 @@ +package mock + +import ( + "context" + "io" + + "github.com/pilosa/pilosa" +) + +var _ pilosa.TranslateStore = (*TranslateStore)(nil) + +type TranslateStore struct { + TranslateColumnsToUint64Func func(index string, values []string) ([]uint64, error) + TranslateColumnToStringFunc func(index string, values uint64) (string, error) + TranslateRowsToUint64Func func(index, frame string, values []string) ([]uint64, error) + TranslateRowToStringFunc func(index, frame string, values uint64) (string, error) + ReaderFunc func(ctx context.Context, off int64) (io.ReadCloser, error) +} + +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + return s.TranslateColumnsToUint64Func(index, values) +} + +func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) { + return s.TranslateColumnToStringFunc(index, values) +} + +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + return s.TranslateRowsToUint64Func(index, frame, values) +} + +func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { + return s.TranslateRowToStringFunc(index, frame, value) +} + +func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { + return s.ReaderFunc(ctx, off) +} diff --git a/pilosa.go b/pilosa.go index c18fb5c0c..bff167a7d 100644 --- a/pilosa.go +++ b/pilosa.go @@ -61,6 +61,8 @@ var ( ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") ErrResizeNotRunning = errors.New("no resize job currently running") + + ErrNotImplemented = errors.New("not implemented") ) // ApiMethodNotAllowedError wraps an error value indicating that a particular @@ -83,6 +85,7 @@ var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) // Can have a set of attributes attached to it. type ColumnAttrSet struct { ID uint64 `json:"id"` + Key string `json:"key,omitempty"` Attrs map[string]interface{} `json:"attrs,omitempty"` } diff --git a/pql/ast.go b/pql/ast.go index c3deff9dd..2d3f59e58 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -40,6 +40,23 @@ func (q *Query) WriteCallN() int { return n } +// HasKeys returns true if any call in the query uses keys and requires translation to ids. +func (q *Query) HasKeys() bool { + for _, call := range q.Calls { + if call.Args["col"] != nil { + if _, ok := call.Args["col"].(string); ok { + return true + } + } + if call.Args["row"] != nil { + if _, ok := call.Args["row"].(string); ok { + return true + } + } + } + return false +} + // String returns a string representation of the query. func (q *Query) String() string { a := make([]string, len(q.Calls)) @@ -100,6 +117,22 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { } } +// StringArg is for reading the value at key from call.Args as a string. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. An error is returned if the value is not a string. +func (c *Call) StringArg(key string) (string, bool, error) { + val, ok := c.Args[key] + if !ok { + return "", false, nil + } + switch tval := val.(type) { + case string: + return tval, true, nil + default: + return "", true, fmt.Errorf("could not convert %v of type %T to string in Call.StringArg", tval, tval) + } +} + // Keys returns a list of argument keys in sorted order. func (c *Call) Keys() []string { a := make([]string, 0, len(c.Args)) diff --git a/row.go b/row.go index b7643a975..a59bd73e0 100644 --- a/row.go +++ b/row.go @@ -27,6 +27,9 @@ import ( type Row struct { segments []RowSegment + // String keys translated to/from segment columns. + Keys []string + // Attributes associated with the row. Attrs map[string]interface{} } @@ -166,6 +169,11 @@ func (r *Row) ClearBit(i uint64) (changed bool) { return s.ClearBit(i) } +// Segments returns a list of all segments in the row. +func (r *Row) Segments() []RowSegment { + return r.segments +} + // segment returns a segment for a given slice. // Returns nil if segment does not exist. func (r *Row) segment(slice uint64) *RowSegment { @@ -241,8 +249,10 @@ func (r *Row) MarshalJSON() ([]byte, error) { var o struct { Attrs map[string]interface{} `json:"attrs"` Columns []uint64 `json:"columns"` + Keys []string `json:"keys,omitempty"` } o.Columns = r.Columns() + o.Keys = r.Keys o.Attrs = r.Attrs if o.Attrs == nil { diff --git a/server.go b/server.go index cbc79a86c..4a453cef5 100644 --- a/server.go +++ b/server.go @@ -52,10 +52,11 @@ type Server struct { closing chan struct{} // Internal - Holder *Holder - Cluster *Cluster - diagnostics *DiagnosticsCollector - executor *Executor + Holder *Holder + Cluster *Cluster + TranslateFile *TranslateFile + diagnostics *DiagnosticsCollector + executor *Executor // External handler Handler @@ -75,6 +76,8 @@ type Server struct { diagnosticInterval time.Duration maxWritesPerRequest int + primaryTranslateStore TranslateStore + defaultClient InternalClient dataDir string } @@ -169,6 +172,13 @@ func OptServerInternalClient(c InternalClient) ServerOption { } } +func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption { + return func(s *Server) error { + s.primaryTranslateStore = store + return nil + } +} + func OptServerStatsClient(sc StatsClient) ServerOption { return func(s *Server) error { s.Holder.Stats = sc @@ -241,6 +251,14 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.Cluster.Logger = s.logger s.Cluster.Holder = s.Holder + // Initialize translation database. + s.TranslateFile = NewTranslateFile() + s.TranslateFile.Path = filepath.Join(path, "keys") + s.TranslateFile.PrimaryTranslateStore = s.primaryTranslateStore + if err := s.TranslateFile.Open(); err != nil { + return nil, err + } + // update URI port with actual listener port. TODO this should probably be done outside of here. if s.URI.Port() == 0 { s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) @@ -259,8 +277,10 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.executor.Holder = s.Holder s.executor.Node = node s.executor.Cluster = s.Cluster + s.executor.TranslateStore = s.TranslateFile s.executor.MaxWritesPerRequest = s.maxWritesPerRequest s.handler.GetAPI().Executor = s.executor + s.handler.GetAPI().TranslateStore = s.TranslateFile return s, nil } @@ -354,6 +374,9 @@ func (s *Server) Close() error { if s.Holder != nil { s.Holder.Close() } + if s.TranslateFile != nil { + s.TranslateFile.Close() + } return nil } diff --git a/server/config.go b/server/config.go index 6c7db569a..1b74b177b 100644 --- a/server/config.go +++ b/server/config.go @@ -78,6 +78,11 @@ type Config struct { // Gossip config is based around memberlist.Config. Gossip gossip.Config `toml:"gossip"` + // Translation config supports translation store replication. + Translation struct { + PrimaryURL string `toml:"primary-url"` + } + AntiEntropy struct { Interval toml.Duration `toml:"interval"` } `toml:"anti-entropy"` diff --git a/server/server.go b/server/server.go index d034a4aa8..0c4184d60 100644 --- a/server/server.go +++ b/server/server.go @@ -217,6 +217,12 @@ func (m *Command) SetupServer() error { c := http.GetHTTPClient(TLSConfig) + // Setup connection to primary store if this is a replica. + var primaryTranslateStore pilosa.TranslateStore + if m.Config.Translation.PrimaryURL != "" { + primaryTranslateStore = http.NewTranslateStore(m.Config.Translation.PrimaryURL) + } + m.Server, err = pilosa.NewServer( pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)), @@ -235,6 +241,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerListener(ln), pilosa.OptServerURI(uri), pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), + pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore), ) return errors.Wrap(err, "new server") diff --git a/statik/statik.go b/statik/statik.go new file mode 100644 index 000000000..54ef98b8f --- /dev/null +++ b/statik/statik.go @@ -0,0 +1,10 @@ +package statik + +import ( + "github.com/rakyll/statik/fs" +) + +func init() { + data := "PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x17\x00\x00\x00assets/chevron-down.png\x89PNG\x0d\n\x1a\n\x00\x00\x00\x0dIHDR\x00\x00\x00\xc8\x00\x00\x00\xc8\x08\x06\x00\x00\x00\xadX\xae\x9e\x00\x00\x0e\x0eIDATx\xda\xed\xddy\x90\x14\xd5\x1d\x07\xf0\xc7\xce\xd5=}\xce\xec\x1c;\xb33\xb3;3{\xc0\x9e\xec\x01\xcb.\xbb\xec1\xbb\xa8A\xa3$h\xc5#\x1e \xb95\xa5Dc*\x95C<\"\xa5\xd1T\x02\xc6Jb\x89\xe6\x1f\xe3\x91hb\"\xa8\x89g\x8c\xa6\"\xc6\x8ax!`\x8c ^ \x88r\xaf\xc9\xef\x07\xa31D\x84\x85\xdd\x9973\xdfOUWQ\xcbL\xf7{\xef\xf7\xeb\xee7\xef\xf5!\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe4\x8d\x93\x16\xc3\x8cu\xfby\xa1\x7f\xeb\xb48\xd0,p\x10\x8e\xfd\xf2\xc6\xc8\xe6RQ\xd1\\V\xbc\xd3\xa1X\x0b\xed\x9a\xcc\xb7yq(\xf6\x99n_\xba\x89\xfe\xcf\x83\x1c\x80\x03\xf0p\x8eP\xae\xcc\x7f?o\xca<\xd6\xe78\x978\xa7\x8a\xa5\x92\x15j\xb0a\x9eQ9m\xb9Z^\xb7\xdeW;\xf2o^\xd4@\xfdsf|\xc62oE\xdb1\xf4\x19\x1fr\x01\xf6\xe3\xe3\xdc\xa0\x1c\xb9\x96r\xe5\x85\x0f\xf2\xa6\xbc\xee\xa5\xbd\xb9D9\xc5\xb9U\xe8\x95\xacR}\xa9\xaf\x1bU=\xcf\xbc_\xc1\xfd\x17\xb3\xba\xef/\xde\xc0\xe4\x85\xf4\xd90r\x02\xb2\xc2\x9c\x13\x9c\x1b\x07\xca\x1b\xce)\xd5\x97\xbe\x90s\xac\x10+\xa8*v\xb2\xca\xa1\x96/;P\x05\xf7_\x9cZ\xf8\"::T\xa2\xcbU\xda]*\xce\x01\xce\x85C\xcd\x1b\xca\xb1\xa5\x8a/\x95\xa0\xef*\x85RIEx\xb4\xd9\xde\xd0\x94{\x0f\xb5\x92\xef/\xde\x8a\xe6[\x85jt\xd3:\xdc\xc8\x95\x92\xe3\xa6\xd8\xf7p\x0e\x8c9o(\xd7(\xe7F\ne'9\xdf\xa8\x9a\xf9\x8a]32:\xd6\x8a\xf2w\xcc\xea\xde\xf5\xb4\x8e3\x91/%\xe7,\x8e\xfd\xe1\xe6\x0d\xe7\x1c\xad\xe3<\x99+\x18sx\xfd\xd7\xd9\xe9\xc1\xcdv\xcd\xf0\x9e\xb1V\xf2\xbf\x95\x1d\xdeC\xeb\xd8\xe4\xd4\x02W\xd2:#\xc8\x9b\xa2\x17\xa1X_\xc51\x1f\x87\xbc\xd9\xcc9\xc8\xb9([%\x8f\xd3\xc2\xcd\xbf\xb1\x92\xb36\x1dn\x05\xf7_\xacd\xff\x9bz\xb8\xf5\x97\xb4\xee\xa3\x91CE\xebh\x8e1\xc7z\xfc\xf2f\xd6&\xceEZ\xf7\xb12T\xb0\x8c\x96\x0b\x8d\xea\x9e\xa7\xecd\xff\xd6\xf1\xaa\xe4\x07G\x85T\xff\xdbf\xf5\xcc'\xb9\xdb\x96\xdd\x16\x14\x07\x8e\xe5\"\x8e-\xc7x\xdc\xf3\x86r\x91s\x92\xb6qA\xbe\xf2f\x12-\x8dn\xbb\xfab\xbd\xb2}-\x9d\xde\xde\x1b\xefJ~\xe8\xd4\xf9\x1emc\x0do\x8b\xb6\xd9\x94\xdd6\x14&\x8e]s6o^\xccA\xde\xac\xcd\xe6Mc.\xf3\x86\xf7\xc8\x11\xad\xa2e\xb9\x11\x9f\xbea\xa2*\xf8\x7f\xe3\xde\xb4-\xda\xe6\x8d\xb4\xed\x13\x04.Q)D|y\xc8\\\x8a\xe1My\xc8\x9b\xe5\x9c\xb3\xb9:\x9b\xd4z\xc3M+\xa9\xdf\xf8N\xae*\xf9\xa1\xfe\xe5\xbbj\xa8\xe1a\xe12\xe6S9\xca\x91s\x05\xa3\x9ccF\xb1{\x84c\x98\xfb\xbc\xe9\x7f\xc7[\xd1\xb4\x82\xca\x91\xceEeO\xceu\x05\xf7_\x94@\xed:\x8f]\xcdCz\xad\xa2\x08/\\+\xb2\xb3F+\xc7\x8ac\x96\xef\xbc\xa1\xb2\x9c\x94\x8bJ\x9f\x91\xef\x8a\xf2\xa2G\xdb7\xba\xcd\x04\x9f:3\x02\xb3\xef2\xe2\x98\x0cs\x8c8V2\xe4\x0c\x95\xe7\xb4\\T\xfdR*[\x1dr\xfb\x88\xd5s[r\x9b\xca\x12_=\xd6\xf9\x00\xe7\x1c\xe7^\xc1\x1ca<\xbe\xf41z\xb4\xfdf:\xfdJ1\x14l\xa7\x07\xff\xa9'\xba~.\x9cJ\xaf\xc0\xec\xfb\xe1\xf5\x10\xa8\xed\xb8\x0d\xb9-%\xe9Fo\xe7\x1c\xe3\\\x93\xb1Ku(\xda\x8dD\xcfO\xadT\xff\x1aY\x8e6\xdeH\xcb\x1djp\xf2\xa7\x05n\xc6\x1a\x8b\x08\xb7\x19\xb7\x9d4\xbd\x82T\xff\x0b\x9c[\x9cc\x85\xde\xb8 \xad\xb2c\x91Y\xdd\xfb'Y\x1a\xd7\x8cM_\xa5\x84\x1b.\xe2\xee\x02r\xff\xa0&s[q\x9bI\x13?\xca%\xce)\xce\xadbid\x8f\xd3\xe3\x9fm\xa5\x06\x1e\xb3k2[\xe48=\x0fn\xf1X\x89\x1f\xbb\xb50\xff\xa8\xf3b?\xf8?^n\x1bj\xa3\xeb\xb8\xad\xe4\x18t\xc9l\xe1\x1c\xe2\\\x12Ez\xfd]\xbd\x91\xec\xbb\xc5J\x0fm\x90\xe5h\xa4\xf8k\xeeq\x9a\x91\xa3\xa8l~\xec\x13\x1f\xf0s\x9b(\xfe\xf4\xbd\x12\xcdmm\xe0\xdc)\x85\x81\x16\x9f\x1an9\x97\x8e\x06odg\xbf\xf3>\xebj\xc4\xa7\xaf+s\x1b\x17e\x87\x82K\xf9\x8eE\xae\xbb\x8f\xdb\x82\xdbD\x92\xab#\xf8\x92\x917\xd4p\xeb9\xa56T?\x85*\xbe\xd5JgFe\xd8I\xf6^\x0e=\xc9\xc9\x0f\x890Kx\x071\xa9\x0d~%\xc9\x1c\xd6{Vjh\x94s\x84\x7f\x07\x95b0\x1c\x1e#Y\xef\xb1\x93?SC\x0d\xb2\xfc.\xd9A\xdd\x8a\x87\xa9l\xc7\x8b\xd2\xba*\x98\xebz<\xd5\xfd\x11n\x03\x19b\xc19\xe11\xa2?\xe1\x1c\x11%|\x85\xb6C\xa8\xfe\xb8C\xf1\x9d\xea\xd4C\xaf\xc8ry\xb4\x11\xebz\x8e\xca\xb6\x84\x16\xb5\x04b\xc0u\\\xc2u\x96\xe5\xf6\x05\xce\x05\xca\x89S(7b\x02\xb7/\xec\xed\xf7Z\xc2\xe9\x99\xed\xd2\xc3\xbf7\xabfn\xcb\xff\xe9}h\xb7\x91\x98\xb1\x81\x82t\x13\x95\x8dG\xb9\x8a\xf1\xf2y\xaeS3\xd7\x91\xeb\xcau\xce\xfb\xf0-\xc5\x9er\xe0>\xca\x85\x91\xbd9\x81'\xd8\xfc\x0f~\x1eo\x1fu\xb9\xae\xf2\x86\x9b\xa4\x18\xe52b\xd3\xb6\xa8\x81\xfa;\xa9\\\xa7\x17\xd9P0\xd7\xe5t\xae\x1b\xd7Q\x8a \\\x8a9\xc7\x9es@\xe0V\x85\x8f=\xaa\xd5\xba\x8c\xd8\x02\xc5\x9fz^\x86\x1f\xeftT\xe3;\xd2VQ\xb9.+\x92\xa1`\xae\xc3e\\'\xae\x9b\x0c?\xc6)\xd6\xcfr\xcc9\xf6\x027\xbb\x1d\x12>\x82,\xd0\"\xad+(\x88\x9b\xa4\xf8\xd1X\x9e\xde\xa8\xf8R\x97;\xfd\xe9\x1eQ\x98\x93T\x8a\x12\xe9\xea\xe5:p]\xa4\x98\x11\xa7\xd8r\x8c\xa9l\xf3q\xd68\xbc\xb3IF\x0dMY\xaaWv\xbe,\xc5U\xa3\x95\x1d\xdb\xf4X\xe7\xedf\xa2\xfb\xf4l\x1f\xb9PX\xf4;\xe3\x0c39\xeb\x0e\xae\x83\x1cm\xd9\xf92\xc7\x96\xca6\x84\xb3\xc6\x91\xa9t*\xfe\xefP\x80\x9f\xb1\xd2C;$\x18\xe5\x1a\xb5\xd3C/\xe9\xd1\xf6/ 5\x1a\x97|\x94\xc5\xc1e\xe4\xb2r\x99%\x99\x11\xdf\xc1\xb1\xa4\x98~K\xe0\x8e\xcfq\x1d\xe9\x9aO\x0d\xfb\x90\x95\xec\x7fK\x9a\xfb\x10\xa2m\x17k\xc1\x86\xa9\x92\x0e\x07\xab\\6=\xd2\xb6X\x9a\xcbE(v\x1cC*\xdbY\x18\xa1\x9a\x18\xd3\x94\xf2\xba\xe5\xd4\xd0\x1b\xe5\xb9\xb2t\xe6\n\x8f?9[\xb2Q.\xaf\xc7_}\x14\x95m\xa5<;\xc7\xc0\x9b\x14\xbb\x1b8\x86H\xe3\x89\xee6\x08\xf1M_:\xf3\x8e];\xbc[\x8e\xcb\"Fv\n\xb7>O\x88j%\xcfG\xc6I{\xcb\xe0\xd6O\xe42I\xd16\x1c#\x8a\x15\x95\xed\x1b\x02\x93~9\xdcI\x1c\x8e\xe3\x8d\xf8\xf4?\xcbr\xdd\x10-\xbb\xd4P\xe3\xd5\"\x87O\xef\xfb\x08I5\xd8x\x0d\x97E\x96\xeb\xdb(F\x8fS\xac\x8e\xc3\xce\x91\x87aK\xbe_A-\xaf[&K7\xc2\xae\xc9\xbcmT\xf5\xf2\xb5\\#yh\x8f\xd9\xbcm.\x83,\xedA\xb1Y\xea\xd6\xc3\xfcN\x17\x05\xe9\x9a\xaf\xf9\x125V\xe9\xb1\x13_\xb5\xd2\x83\xafK\xb2\x93l\xb7R\x03k\xca\xbc\xe5_\x16\xb9\xb92\xd8\xe4m\xf16y\xdb\x92\\\xf4\xf9:\xc5\xe4\\\xe1\x8d\xf3(\x15\xee\xff\x97@P 5\x9f\xaa'\xbaWH\xf3\xe3\xbd\xaa\xf7E-\xdcr\x0d\x95\xadc\x02\xeb\xdd\xc9\xdb\xe0m\xc9Ro\xa3\xaa\xfbn\x8e\x05\xc7\x04i)\x17C\x89\xb4\xf6j\x91\xf6k$\x1a\xd6|\xd5H\xcc\xf8-\x95m\xde\x04\xd4w\x9e\x11\x9f\xf1;\xde\x864O4\x8c\xb4_\xcd1\xe0X \x1d\xa5\x9d/\xf1\xa4\x8dD\xcf\x12:\xcd\xaf\x91\xa4\xcb\xb5K\x8b\xb4>\xe62bgS\xf9\xe2\xe3P\xc78\xadk!\xaf\x93\x1f\xce,I\x97\x8a\x9f0\xb2\x84\xda>\x85\xf9\x8d\x02\xe9r\xe9\x89\x19\xe7\xd0\xd1u\xa5<\x93\x8a\xed\xcf{\x83\x0d|\xc1c\xf7a&\x11\x7f\xa7\x9b\xd6q9\xad\xeb\x05\x89\xce\x92+\xf5\xaa\xee\xaf\x08\x89\x9e\x83\x0b\x878\xca\xa5T4\x0eZ\xa9\xfe\xdb\xcdD\xcf\xdbv:3*\xc1\xe3j6\xa9\xe5\xb5\xb7 \xa7\x87\xdf\xfd>\x96\x89E/\x7f\x87\xbf\xcb\xeb\xc8\xffC\xf82\xa3Fl\xda[Vj\xe06\xa5\xa2i\x00\xa3T\x85-\xa2\xf8jn\xd2+;\x9e\xa5\xe4\x92b\xf2\xccm'V\xb9\xed\xd8\xdcl\x97k\xd2A\xce\x1aq\xfe,}\xe7II\x9eI\xb5\x93\xdb\xd2\xa5\x85\xaf\xa7\xb2\x85\x91^\xc5\x81\xdf\x11\xb1P\xf1\xa7\x1f5\xaaz\xa4\x18\n\xf5V4\xff\xc3\xa9\x07/\xa1rM\x11\x1f=\x89\xc6\x7f\x9bB\x9f\xb9\x94?+\xc7\x08U\xcf6j\xc3?R\xb9\x16d\xdb\x14\x8a\x08_R]/\x1c\xca\x83vr`\xab\x0c].\xea\xfe\xed\x98\xe4R\xff N\x97Z\xd1z\x85\x11\xefZ]*;\x07\xd5\xf5i5\xd2\xfa=\x81\xdba\xe1\x10\xc5\x14\x7f\xf2l\xbd\xb2\x83\xdf\xe2\xba\xa7Xw\x0c\xae\x9b\x1e\xebxH\xf1\xa7\xf8\xb9T1\x84\x1d\xc6j\x8eY5\xf3\x1e+5\xf0j\xd1u\xa9\xa8N\xfcP\x07\xaa\xe31\x083\x1c\x89\x16o\xb4\xe3\x87\xd4G_[<;\xc7\xac\x17\xb5\xca\x0e\x9e\xf8kFxa<\x98.#r\xb6\x9d\x1a\\O\xdd\x92\x1d\x05\xdc\xa5\xda\xc1up\x19Q\xeeR\x99\x08+\x8c\xb7\x99fr\xd6\xe3\xbe\x9a\xcc\xb6\x82\xdbA\xa8\xcc\\v\xb1\xef~\x14\x80 \x93\xd0\"m\xdf/\xb8\xc9\xbfH\x1b\xbfZ \x8e\xf0\xc1D+\x13\x81z\xc3\xe1\x0d|\xb2Pv\x0e*\xeb\x1c.\xb3\xc0C\xa2!\x87\xbcZ\xb0\xe1h\xfa\xf1~\x9f\xbc\x93\x7f\xfd\xf7j\xa1&\xd9\x1e\x8b\n%D\xf1VL\x9b\xa6\xc7g\\\xa2E;\xb6\xcb\xb2cx\xc3M[\xf5x\xf7bodz\xa7(\xccw\x99@\x11q(\xbe\x86\x84\xc7W\xfb\x05\xb7\x9dXMg\x94]y\xbc\\d\x97\xdb\x8e\xff\x8d\x9fz\xa2\xfa\x9bd\x7f5\x03\x94\x18\xbe\x99\xe8\x145P\xf3\x0b=\xda\x96\xf3\xf7\xfe\xe9\x95m\x9bi\xdb7S\x19>C\x8b\x86p\x80\x8c\xf8E\xa4\xddn+~\xb9\x1a\x9c\x92\xb3\xd9w\xda\xd6\x06\xda\xe6\xa5\xb4\xed.\x81W\x99A\x01\xf0 Q\xb6X\xaf\xec|\xd2\xac\xee\x9b\xb09\x13\xab\xba\xef]\xda\xc6*\xda\xd6wi\x9b6\x9a\x1d\n\xedl\xf2Yo\xb0\xf1\xd7f\xa2{\xdc\xdf\x8aE\xeb\xdc\xe4\x0d5\xf2+\xaaO\xcbn\x0b\xa0\xe0\xf0\xbcC\xa3K\x0b\xde`V\xf5\xbcj\xa73{\xc6\xe1 #{x]\xb4N~.U\x83\xc0\xdc\x06\x14\x01\xfe]p\x99Y5s\xed\x91\xbc\x88\x94\xbe\xbb\x9d\xd7A\xeb\xe2\xdba\xf1Z\x01(*|o\xf7\xb1j\xb0\x9e_r\xb3k\xec\x17\x1afv\xa9\xc1\xc9\x0f\xd2:\xe6\xa0)\xa1X9\x84\xd0\x83en\xed\xc6\xb1\xee \xf4\x9d\xeb\xf9\xbb\x02s\x1bP\x02g\x92r\xa7\xea_d$\xba\x0f\xfa\xe2\x1b\xfa\xcc\x1a\x97\xea?\x8f\xbf#\xf0\x84\x11(!\x01\x8f\xafj\x8e7\xd8p\xd7\x01/\x19 5\xdeI\x9f\xf9Dv\xe7\x00(9\x9aK\xabl\xd5BM\x8b= 0) {\n output_string += `
\n
\n Just getting started? Try this:
\n :create index test
\n :use test
\n :create frame foo
\n SetBit(rowID=0, columnID=0, frame=foo) # Use PQL to set a bit\n `\n }\n }\n }\n }\n\n\n var markup =`\n
\n
\n
\n
\n
Input
\n       \n Source: ${res.indexname}\n
\n
\n ${res.input}\n
\n
\n
\n
\n
output
\n       \n ${res.querytime_ms} ms\n
\n
\n ${output_string}\n
\n
Expand
\n \n
\n
\n
\n \n
\n
\n `\n node.innerHTML = markup;\n this.output.insertBefore(node, this.output.firstChild);\n\n // Expand when overflow\n var element = this.output.firstChild.getElementsByClassName(result_class)[0];\n var expand = this.output.firstChild.getElementsByClassName(\"expand\")[0];\n if (element.clientHeight < element.scrollHeight) {\n expand.style.display = 'block';\n } else {\n expand.style.display = 'none';\n }\n expand.onclick = function () {\n element.style.height = element.scrollHeight + \"px\";\n expand.style.display = 'none';\n return false;\n };\n }\n\n populate_index_dropdown() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/schema')\n var select = document.getElementById('index-dropdown')\n\n xhr.onload = function() {\n var schema = JSON.parse(xhr.responseText)\n for(var i=0; i 0) {\n select.value = 1;\n }\n }\n xhr.send(null)\n }\n\n}\n\nfunction populate_version() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/version')\n var node = document.getElementById('server-version')\n\n xhr.onload = function() {\n var version = JSON.parse(xhr.responseText)['version']\n var version_major_minor = /(v\\d+\\.\\d+)/.exec(version)[0]\n var doc_link = document.getElementById('nav-documentation')\n doc_link.onclick = function() {\n window.open('https://www.pilosa.com/docs/' + version_major_minor + '/introduction/')\n }\n node.innerHTML = version\n }\n xhr.send(null)\n}\n\nfunction handle_nav_click(e) {\n // e.id = \"nav-xxx\"\n name = e.id.substring(4)\n set_active_pane_by_name(name)\n window.location.hash = name\n}\n\nfunction set_active_pane_by_name(name) {\n // toggle the nav buttons\n document.getElementsByClassName(\"nav-active\")[0].classList.remove(\"nav-active\")\n document.getElementById(\"nav-\" + name).classList.add(\"nav-active\")\n\n // toggle the main interface content divs\n document.getElementsByClassName(\"interface-active\")[0].classList.remove(\"interface-active\")\n document.getElementById('interface-' + name).classList.add(\"interface-active\")\n\n // hack hack\n switch(name) {\n case \"cluster\":\n update_cluster_status()\n break\n case \"documentation\":\n open_external_docs()\n break\n }\n}\n\n\nfunction update_cluster_status() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/status')\n status_node = document.getElementById('status')\n xhr.onload = function() {\n var status = JSON.parse(xhr.responseText)\n render_status(status)\n }\n xhr.send(null)\n}\n\nfunction render_status(status) {\n // render node table\n var nodes_div = document.getElementById(\"status-nodes\")\n while (nodes_div.firstChild) {\n nodes_div.removeChild(nodes_div.firstChild);\n }\n\n var nodes = status[\"status\"][\"Nodes\"]\n table = document.createElement(\"table\")\n tbody = document.createElement(\"tbody\")\n table.appendChild(tbody)\n var caption = document.createElement(\"caption\")\n caption.innerHTML = \"(\" + nodes.length + \")\"\n table.appendChild(caption)\n\n var header = document.createElement('tr')\n markup = `Host\n State`\n header.innerHTML = markup\n tbody.appendChild(header)\n for(var n=0; n${nodes[n][\"Host\"]}\n ${nodes[n][\"State\"]}`\n row.innerHTML = markup\n tbody.appendChild(row)\n }\n nodes_div.appendChild(table)\n\n // render index tables\n var indexes_div = document.getElementById(\"status-indexes\")\n while (indexes_div.firstChild) {\n indexes_div.removeChild(indexes_div.firstChild);\n }\n\n var indexes = nodes[0][\"Indexes\"] // TODO currently comes from only node 0\n for(var n=0; nName\n Cache Type\n Cache Size`\n header.innerHTML = markup\n tbody.appendChild(header)\n\n var frames = indexes[n][\"Frames\"]\n if(frames) {\n for(var m=0; m${frames[m][\"Name\"]}\n ${frames[m][\"Meta\"][\"CacheType\"]}\n ${frames[m][\"Meta\"][\"CacheSize\"]}`\n tbody.appendChild(row)\n }\n }\n indexes_div.appendChild(table)\n }\n\n // render slice tables\n // TODO enable when Slices element is present in status response\n /*\n var slices_div = document.getElementById(\"status-slices\")\n data = \"\"\n for(var n=0; n\"\n }\n }\n slices_div.innerHTML = data\n */\n\n}\n\nfunction open_external_docs() {\n window.open(\"https://www.pilosa.com/docs\");\n}\n\nfunction check_anchor_uri() {\n var pane_names = {\"console\": 0, \"cluster\": 0, \"documentation\": 0}\n var anchor = window.location.hash.substr(1);\n if(anchor in pane_names) {\n set_active_pane_by_name(anchor)\n }\n}\n\nDate.prototype.today = function () {\n return this.getFullYear() +\"/\"+ (((this.getMonth()+1) < 10)?\"0\":\"\") + (this.getMonth()+1) +\"/\"+ ((this.getDate() < 10)?\"0\":\"\") + this.getDate();\n}\n\nDate.prototype.timeNow = function () {\n return ((this.getHours() < 10)?\"0\":\"\") + this.getHours() +\":\"+ ((this.getMinutes() < 10)?\"0\":\"\") + this.getMinutes() +\":\"+ ((this.getSeconds() < 10)?\"0\":\"\") + this.getSeconds();\n}\n\npopulate_version()\n\n\nclass Autocompleter {\n constructor(input, output) {\n this.input = input\n this.output = output\n this.keyword_map = this.static_keywords\n this.init_dynamic_keywords()\n }\n\n get static_keywords() {\n return {\n // keyword: length of substring that comes after cursor\n \"SetBit()\": 1,\n \"ClearBit()\": 1,\n \"SetRowAttrs()\": 1,\n \"SetColumnAttrs()\": 1,\n \"Bitmap()\": 1,\n \"Union()\": 1,\n \"Intersect()\": 1,\n \"Difference()\": 1,\n \"Count()\": 1,\n \"Range()\": 1,\n \"TopN()\": 1,\n \"frame=\": 0,\n }\n }\n\n complete() {\n var completer = this\n // extract word fragment ending at cursor. a word fragment:\n // - starts with last nonalpha character before cursor (or beginning of string)\n // - ends at cursor\n var word_start = completer.input.selectionEnd-1\n while(word_start>0) {\n var c = completer.input.value.charCodeAt(word_start)\n if(!((c>64 && c<91) || (c>96 && c<123))) {\n word_start++\n break\n }\n word_start--\n }\n var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd)\n\n // check for keyword match and insert if exactly one match\n var matches = []\n for(var keyword in this.keyword_map) {\n if(keyword.startsWith(input_word)){\n matches.push(keyword)\n }\n }\n if(matches.length > 1) {\n // completer.output.innerHTML = whatever\n }\n\n if(matches.length == 1) {\n // completer.output.innerHTML = \"\"\n var cursor_pos = completer.input.selectionEnd\n var completion = matches[0].substring(input_word.length)\n var before = completer.input.value.substring(0, cursor_pos)\n var after = completer.input.value.substring(cursor_pos)\n completer.input.value = before + completion + after\n var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]]\n completer.input.setSelectionRange(new_pos, new_pos)\n }\n }\n\n init_dynamic_keywords() {\n // hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list\n }\n\n add_keyword() {\n // call when index or frame created in webui\n }\n\n remove_keyword() {\n // call when index or frame deleted in webui\n // issue: if e.g. multiple indexes have same frame, removing one removes all.\n // solution: maintain count. requires more elaborate representation of keywords.\n }\n}\n\nvar input = document.getElementById('query')\nvar output = document.getElementById('outputs')\nvar button = document.getElementById('query-btn')\nvar autocomplete_output = document.getElementById('autocomplete-container')\n\nautocompleter = new Autocompleter(input, autocomplete_output)\nrepl = new REPL(input, output, button, autocompleter)\nrepl.populate_index_dropdown()\nrepl.bind_events()\n\ninput.focus()\n\ncheck_anchor_uri()\n\nfunction isJSON(str) {\n try {\n JSON.parse(str)\n } catch (e) {\n return false\n }\n return true\n}\n\nfunction parse_query(query, indexname) {\n var keys = query.replace(/\\s+/g, \" \").split(\" \");\n var command = keys[0];\n var command_type = keys[1];\n var command_name = keys[2];\n var option_str = keys.slice(3, keys.length)\n var options = parse_options(option_str);\n if (command !== \":use\") {\n if (!command_name){\n return {}\n }\n }\n\n var parsed_query = {};\n parsed_query[\"command\"] = command.substr(1, command.length);\n parsed_query[\"command_name\"] = command_name;\n switch (command) {\n case \":create\":\n parsed_query[\"request\"] = \"POST\";\n if(Object.keys(options).length === 0) {\n parsed_query[\"data\"] = \"\";\n } else {\n var opts = {\"options\":{}};\n for (var o in options) {\n opts.options[o] = options[o]\n }\n parsed_query[\"data\"] = JSON.stringify(opts);\n }\n switch (command_type){\n case \"index\":\n parsed_query[\"url\"] = '/index/' + command_name;\n break;\n case \"frame\":\n parsed_query[\"url\"] = '/index/' + indexname + '/frame/' + command_name;\n break\n }\n break;\n case \":delete\":\n parsed_query[\"request\"] = \"DELETE\";\n switch (command_type){\n case \"index\":\n parsed_query[\"url\"] = '/index/' + command_name;\n parsed_query[\"data\"] = \"\";\n break;\n case \"frame\":\n parsed_query[\"url\"] = '/index/' + indexname + '/frame/' + command_name;\n parsed_query[\"data\"] = \"\";\n break;\n }\n break;\n case \":use\":\n parsed_query[\"command_name\"] = keys[1];\n break;\n default:\n return {}\n }\n return parsed_query;\n}\n\nfunction parse_options(option_str) {\n var int_keys = [\"cacheSize\"];\n var bool_keys = [\"inverseEnabled\"];\n var options = {};\n for (var i = 0; i < option_str.length; i++) {\n var parts = option_str[i].split('=');\n if (int_keys.indexOf(parts[0]) !== -1 ){\n options[parts[0]] = Number(parts[1])\n } else if (bool_keys.indexOf(parts[0]) !== -1){\n options[parts[0]] = (parts[1] == \"true\")\n } else {\n options[parts[0]] = parts[1]\n }\n }\n return options;\n}PK\x07\x08\xfa\x8b=\x1a\xcaH\x00\x00\xcaH\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\x00\x00\x00assets/nav-cluster-active.svgnav_cluster_1\nPK\x07\x08\xc1J\xead \x02\x00\x00 \x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00assets/nav-cluster.svgnav_cluster_1PK\x07\x08\xc4\x07\xec\x0b\x05\x02\x00\x00\x05\x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\x00\x00\x00assets/nav-console-active.svgnav_consolePK\x07\x08\xf2\x90\xe75\xa0\x01\x00\x00\xa0\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00assets/nav-console.svgnav_console\nPK\x07\x08\xfb\xc8\xea\xb0\x9e\x01\x00\x00\x9e\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00#\x00\x00\x00assets/nav-documentation-active.svgdocumentation\nPK\x07\x08\xe5\x95\x86\x82\xec\x01\x00\x00\xec\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00assets/nav-documentation.svgdocumentationPK\x07\x08\xe18\x81J\xe8\x01\x00\x00\xe8\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00assets/nav_item1.svgnav_item1PK\x07\x08+\xd4\xf31\xa2\x01\x00\x00\xa2\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xa0~\xe6J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00assets/style.css*{\n -webkit-box-sizing: border-box;\n -moz-box-sizing: border-box;\n box-sizing: border-box;\n}\nbody{\n font-family: sans-serif;\n background-color: #fbfcfd;\n margin: 0;\n color: #102445;\n}\nh2{\n margin-bottom: 30px;\n}\n\nh5{\n text-transform: uppercase;\n letter-spacing: 2px;\n line-height: 1.21;\n margin: 0;\n}\na{\n line-height: 1.38;\n letter-spacing: 0.2px;\n text-decoration: none;\n color: #102445;\n}\n\n\na:hover{\n color: #1db598;\n}\n\ntextarea{\n width: 100%;\n margin-bottom: 10px;\n border-radius: 2px;\n background-color: #fbfcfd;\n border: solid 1.5px #e4eff4;\n font-family: monospace;\n font-size: 16px;\n line-height: 1.5;\n letter-spacing: 1.1px;\n outline: none;\n padding: 30px;\n}\n\n\nselect{\n /*-webkit-appearance: none;\n -moz-appearance: none;\n appearance: none;\n background: url(\"img/chevron-down.png\") no-repeat calc(100% - 10px) !important;*/\n border-radius: 3px;\n background-color: #fbfcfd;\n width: 187px;\n height: 50px;\n border: solid 1.5px #e4eff4;\n font-size: 18px;\n font-weight: bold;\n line-height: 1.39;\n letter-spacing: 0.2px;\n color: #102445;\n padding: 10.5px;\n\n}\n\nbutton{\n width: 165px;\n height: 50px;\n border-radius: 3px;\n background-color: #1db598;\n outline: none;\n border: none;\n font-size: 16px;\n color: white;\n}\n\nem{\n font-style: normal;\n opacity: 0.5;\n font-size: 14px;\n font-weight: 500;\n letter-spacing: 0.2px;\n color: #102445;\n}\n\n.header{\n height: 92px;\n display: flex;\n align-items: center;\n justify-content: space-between;\n width: 90%;\n margin: auto;\n}\n\n.container{\n display: flex;\n height:100%;\n min-height: 100vh;\n}\n.nav{\n color: white;\n display: flex;\n flex-direction: column;\n width: 150px;\n background: #3c5f8d;\n}\n\n.nav-item{\n height:150px;\n display: flex;\n align-items: center;\n justify-content: center;\n flex-direction: column;\n border-bottom: 3px solid #2a4871;\n cursor: pointer;\n}\n\n.nav-active{\n background: #f2f7f9;\n font-weight: bold;\n color: #1db598;\n}\n\n.nav-item > .nav-image {\n display: flex;\n}\n\n.nav-item > .nav-image-active {\n display: none;\n}\n\n.nav-active > .nav-image {\n display: none;\n}\n\n.nav-active > .nav-image-active {\n display: flex;\n}\n\n\n.interface{\n display: none;\n flex: 1;\n flex-direction: column;\n align-items: center;\n background: #f2f7f9;\n}\n\n.interface-active{\n display: flex;\n}\n\n.query{\n margin-bottom: 30px;\n}\n.query,\n.output-container,\n.status-container{\n width: 75%;\n}\n\n.output{\n margin-bottom: 30px;\n}\n\n.input-controls{\n display: flex;\n justify-content: flex-end;\n}\n\n.tabs{\n display: flex;\n background: #eaf2f6;\n}\n.active-tab{\n background: white;\n font-weight: bold;\n color: #1db598;\n\n}\n\n.tab{\n height:60px;\n width: 100px;\n border-top-right-radius: 5px;\n display: flex;\n align-items: center;\n justify-content: center;\n visibility: visible;\n cursor: pointer;\n\n}\n\n.pane{\n background: white;\n padding: 30px;\n display: none;\n}\n\n.active{\n display: block;\n}\n\n.result-io-header{\n display: flex;\n align-items: center;\n margin-bottom: 15px;\n}\n\n.result-input,\n.result-output,\n.result-error{\n height: 60px;\n border-radius: 2px;\n background-color: #fafafa;\n border: solid 1.5px #e4eff4;\n font-family: monospace;\n font-size: 16px;\n line-height: 1.5;\n letter-spacing: 1.1px;\n color: #102445;\n padding: 15px;\n margin-bottom: 15px;\n word-break: break-all;\n overflow-wrap: break-word;\n overflow:hidden;\n}\n\n\n.result-output{\n background-color: #edf9f7;\n border-left: solid 4px #1db598;\n}\n\n.result-error{\n background-color: #fbf1f0;\n border-left: solid 4px #fa3035;\n color: #fa3035;\n}\n\n.raw{\n height: 253px;\n display: flex;\n align-items: center;\n justify-content: center;\n}\n\n\n.result-table > table {\n border-left: solid 4px #1db598;\n}\n\ntable{\n border: solid 0.5px #e0e0e0;\n width: 100%;\n margin-bottom: 30px;\n /*color:#3c5f8d;*/\n}\ncaption{\n text-align:left;\n font-size: 16px;\n font-weight: bold;\n line-height: 1.21;\n letter-spacing: 2px;\n text-align: left;\n}\nth{\n font-size: 14px;\n font-weight: bold;\n line-height: 1.21;\n letter-spacing: 2px;\n color: #102445;\n text-transform: uppercase;\n text-align: left;\n padding: 21px 30px;\n background-color: white;\n}\ntr{\n border: solid 0.5px #e0e0e0;\n background-color: white;\n}\ntr:nth-child(even) {\n background-color: #f2f7f9;\n}\ntd{\n padding: 21px 30px;\n}\n\n.expand {\n text-align: center;\n}\n\n.query h2 {\n display: inline-block;\n}\n\n.query-tooltip {\n position: relative;\n display: inline;\n color: #000;\n margin-left: 5px;\n}\n\n.query-tooltip:hover {\n color: #000;\n}\n\n.query-tooltip-content {\n background-color: rgb(250, 250, 250);\n border: solid 1.5px #e4eff4;\n color: #102445;\n border-radius: 2px;\n padding: 15px;\n margin-bottom: 15px;\n\n position: absolute;\n left: 80px;\n top: -30px;\n z-index: 1;\n}\n\n.query-tooltip-container {\n position: relative;\n visibility: hidden;\n}\n\n.query-tooltip:hover+.query-tooltip-container{\n visibility: visible;\n}\n\n.code{\n font-family: monospace;\n}\n\nPK\x07\x08\xec[\xd0\xfe=\x13\x00\x00=\x13\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00cz\xbfJ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\x00\x00\x00index.html\n\n\n \n \n \n \n Pilosa WebUI\n \n\n\n
\n \"\"\n
\n
\n
\n
\n
\n \"\"\n \"\"\n Console\n
\n
\n \"\"\n \"\"\n Cluster Admin\n
\n
\n \"\"\n \"\"\n Documentation\n
\n
\n
\n\n
\n

Query

\n ?\n
\n
\n
PQL
\n
\n SetBit(frame=foo, rowID=0, columnID=0)
\n ClearBit(frame=foo, rowID=0, columnID=0)
\n SetRowAttrs(frame=foo, rowID=0, color=\"blue\")
\n SetColumnAttrs(frame=foo, columnID=0, shape=\"circle\")
\n Bitmap(frame=foo, rowID=0)
\n Range(frame=foo, rowID=0, start=\"2010-01\", end=\"2017-03\")
\n Count(<BITMAP_CALL>)
\n TopN([BITMAP_CALL], frame=foo, n=20)
\n Union([BITMAP_CALL, ...])
\n Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
\n Difference(<BITMAP_CALL>, <BITMAP_CALL>)\n
\n
\n
Special commands
\n
\n :create index test [columnLabel=column]
\n :use test
\n :create frame foo [rowLabel=row]
\n :delete index test
\n :delete frame foo\n
\n
\n <tab>: autocomplete
\n <up>/<down>: history
\n
\n
\n \n
\n
\n \n    \n \n
\n
\n
\n\n
\n

Output

\n
\n \n
\n
\n\n
\n\n
\n
\n

Nodes

\n
\n
\n
\n
\n

Indexes

\n
\n
\n
\n
\n \n
\n\n
\n\n
\n docs!\n
\n\n
\n \n\n\nPK\x07\x08\x8dC\xf8\xe1\xef\x0f\x00\x00\xef\x0f\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3JJ\x1c\xff\xa8G\x0e\x00\x00G\x0e\x00\x00\x17\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x00assets/chevron-down.pngPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x96\x84jK\xfa\x8b=\x1a\xcaH\x00\x00\xcaH\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x8c\x0e\x00\x00assets/main.jsPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xc1J\xead \x02\x00\x00 \x02\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x92W\x00\x00assets/nav-cluster-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xc4\x07\xec\x0b\x05\x02\x00\x00\x05\x02\x00\x00\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xe6Y\x00\x00assets/nav-cluster.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xf2\x90\xe75\xa0\x01\x00\x00\xa0\x01\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81/\\\x00\x00assets/nav-console-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xfb\xc8\xea\xb0\x9e\x01\x00\x00\x9e\x01\x00\x00\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x1a^\x00\x00assets/nav-console.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xe5\x95\x86\x82\xec\x01\x00\x00\xec\x01\x00\x00#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xfc_\x00\x00assets/nav-documentation-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xe18\x81J\xe8\x01\x00\x00\xe8\x01\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x819b\x00\x00assets/nav-documentation.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J+\xd4\xf31\xa2\x01\x00\x00\xa2\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81kd\x00\x00assets/nav_item1.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xa0~\xe6J\xec[\xd0\xfe=\x13\x00\x00=\x13\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81Of\x00\x00assets/style.cssPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00cz\xbfJ\x8dC\xf8\xe1\xef\x0f\x00\x00\xef\x0f\x00\x00\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xcay\x00\x00index.htmlPK\x05\x06\x00\x00\x00\x00\x0b\x00\x0b\x00\xf2\x02\x00\x00\xf1\x89\x00\x00\x00\x00" + fs.Register(data) +} diff --git a/test/executor.go b/test/executor.go index c04f91eca..c4af65b01 100644 --- a/test/executor.go +++ b/test/executor.go @@ -20,6 +20,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/inmem" "github.com/pilosa/pilosa/pql" ) @@ -42,6 +43,7 @@ func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster + e.TranslateStore = inmem.NewTranslateStore() e.Node = cluster.Nodes[0] return e } diff --git a/translate.go b/translate.go new file mode 100644 index 000000000..398055fdb --- /dev/null +++ b/translate.go @@ -0,0 +1,1006 @@ +package pilosa + +import ( + "bufio" + "bytes" + "context" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/cespare/xxhash" +) + +const ( + LogEntryTypeInsertColumn = 1 + LogEntryTypeInsertRow = 2 +) + +const ( + DefaultMapSize = 10 * (1 << 30) // 10GB + + DefaultReplicationRetryInterval = 1 * time.Second +) + +const ( + ReplicationBufferSize = 65536 +) + +var ( + ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") + ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") + ErrReplicationNotSupported = errors.New("pilosa: replication not supported") + ErrTranslateStoreReadOnly = errors.New("pilosa: operation not supported, translate store read only") +) + +// TranslateStore is the storage for translation string-to-uint64 values. +type TranslateStore interface { + TranslateColumnsToUint64(index string, values []string) ([]uint64, error) + TranslateColumnToString(index string, values uint64) (string, error) + + TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) + TranslateRowToString(index, frame string, values uint64) (string, error) + + // Returns a reader from the given offset of the raw data file. + // The returned reader must be closed by the caller when done. + Reader(ctx context.Context, off int64) (io.ReadCloser, error) +} + +// Ensure type implements interface. +var _ TranslateStore = &TranslateFile{} + +// TranslateFile is an on-disk storage engine for translating string-to-uint64 values. +type TranslateFile struct { + mu sync.RWMutex + data []byte + file *os.File + w *bufio.Writer + n int64 + writeNotify chan struct{} + + once sync.Once + wg sync.WaitGroup + closing chan struct{} + + cols map[string]*index + rows map[frameKey]*index + + Path string + MapSize int + + // If non-nil, data is streamed from a primary and this is a read-only store. + PrimaryTranslateStore TranslateStore + + // Delay after attempting to connect to a primary that the store will retry. + ReplicationRetryInterval time.Duration +} + +// NewTranslateFile returns a new instance of TranslateFile. +func NewTranslateFile() *TranslateFile { + return &TranslateFile{ + writeNotify: make(chan struct{}), + closing: make(chan struct{}), + cols: make(map[string]*index), + rows: make(map[frameKey]*index), + + MapSize: DefaultMapSize, + + ReplicationRetryInterval: DefaultReplicationRetryInterval, + } +} + +func (s *TranslateFile) Open() (err error) { + // Open writer & buffered writer. + if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { + return err + } else if s.file, err = os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666); err != nil { + return err + } + s.w = bufio.NewWriter(s.file) + + // Memory map data file. + if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.MapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { + return err + } + + // Replay the log. + if err := s.replayEntries(); err != nil { + return err + } + + // Stream from primary, if available. + if s.PrimaryTranslateStore != nil { + s.wg.Add(1) + go func() { defer s.wg.Done(); s.monitorReplication() }() + } + + return nil +} + +func (s *TranslateFile) Close() (err error) { + s.once.Do(func() { + close(s.closing) + + if s.file != nil { + if e := s.file.Close(); e != nil && err == nil { + err = e + } + } + if s.data != nil { + if e := syscall.Munmap(s.data); e != nil && err == nil { + err = e + } + } + }) + s.wg.Wait() + return err +} + +// Closing returns a channel that is closed when the store is closed. +func (s *TranslateFile) Closing() <-chan struct{} { + return s.closing +} + +// Size returns the number of bytes in use in the data file. +func (s *TranslateFile) Size() int64 { + s.mu.RLock() + n := s.n + s.mu.RUnlock() + return n +} + +// IsReadOnly returns true if this store is being replicated from a primary store. +func (s *TranslateFile) IsReadOnly() bool { + return s.PrimaryTranslateStore != nil +} + +// WriteNotify returns a channel that is closed when a new entry is written. +func (s *TranslateFile) WriteNotify() <-chan struct{} { + s.mu.RLock() + ch := s.writeNotify + s.mu.RUnlock() + return ch +} + +func (s *TranslateFile) appendEntry(entry *LogEntry) error { + offset := s.n + + // Append entry to the end of the WAL. + n, err := entry.WriteTo(s.w) + if err != nil { + return err + } else if err := s.w.Flush(); err != nil { + return err + } + + // Move position forward. + s.n += n + + // Apply the entry to the current state. + if err := s.applyEntry(entry, offset); err != nil { + return err + } else if err := s.file.Sync(); err != nil { + return err + } + + // Notify others of write update. + close(s.writeNotify) + s.writeNotify = make(chan struct{}) + + return nil +} + +func (s *TranslateFile) applyEntry(entry *LogEntry, offset int64) error { + // Move offset to the start of the id/key pairs. + offset += entry.HeaderSize() + + var idx *index + switch entry.Type { + case LogEntryTypeInsertColumn: + idx = s.col(string(entry.Index)) + + case LogEntryTypeInsertRow: + idx = s.row(string(entry.Index), string(entry.Frame)) + + default: + return fmt.Errorf("enterprise.TranslateFile.applyEntry(): unknown log entry type: 0x%20x", entry.Type) + } + + // Insert id/key pairs into index. + for i, id := range entry.IDs { + key := entry.Keys[i] + + // Determine key offset based on ID size. + sz := int64(UvarintSize(id)) + idx.insert(id, offset+sz) + + // Move sequence forward. + if id > idx.seq { + idx.seq = id + } + + // Move offset forward. + offset += sz + int64(UvarintSize(uint64(len(key)))) + int64(len(key)) + } + + return nil +} + +func (s *TranslateFile) replayEntries() error { + // Build a reader from the memory-map data. + fi, err := os.Stat(s.Path) + if err != nil { + return err + } + r := bytes.NewReader(s.data[:fi.Size()]) + + // Iterate over each entry and reapply. + for { + offset := s.n + + var entry LogEntry + if n, err := entry.ReadFrom(r); err == io.EOF { + return nil + } else if err != nil { + return err + } else { + s.n += n + } + + if err := s.applyEntry(&entry, offset); err != nil { + return err + } + } +} + +// monitorReplication is executed in a separate goroutine and continually streams +// from the primary store until this store is closed. +func (s *TranslateFile) monitorReplication() { + // Create context that will cancel on close. + ctx, cancel := context.WithCancel(context.Background()) + go func() { <-s.closing; cancel() }() + + // Keep attempting to replicate until the store closes. + for { + if err := s.replicate(ctx); err != nil { + log.Printf("pilosa: replication error: %s", err) + } + + select { + case <-s.closing: + return + case <-time.After(s.ReplicationRetryInterval): + log.Printf("pilosa: reconnecting to primary replica") + } + } +} + +func (s *TranslateFile) replicate(ctx context.Context) error { + off := s.Size() + + // Connect to remote primary. + log.Printf("pilosa: replicating from offset %d", off) + rc, err := s.PrimaryTranslateStore.Reader(ctx, off) + if err != nil { + return err + } + defer rc.Close() + + // Wrap in bufferred I/O so it implements io.ByteReader. + bufr := bufio.NewReader(rc) + + // Continually read new entries from primary and append to local store. + for { + // Read next available entry. + var entry LogEntry + if _, err := entry.ReadFrom(bufr); err == io.EOF { + return nil + } else if err != nil { + return err + } + + // Write to local store. + if err := s.appendEntry(&entry); err != nil { + return err + } + } +} + +func (s *TranslateFile) col(index string) *index { + idx := s.cols[index] + if idx == nil { + idx = newIndex(s.data) + s.cols[index] = idx + } + return idx +} + +func (s *TranslateFile) row(index, frame string) *index { + idx := s.rows[frameKey{index, frame}] + if idx == nil { + idx = newIndex(s.data) + s.rows[frameKey{index, frame}] = idx + } + return idx +} + +// TranslateColumnsToUint64 converts values to a uint64 id. +// If value does not have an associated id then one is created. +func (s *TranslateFile) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // Return error if not all values could be translated and this store is read-only. + if s.IsReadOnly() { + return ret, ErrTranslateStoreReadOnly + } + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.cols[index] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create index map if it doesn't exists. + if idx == nil { + idx = newIndex(s.data) + s.cols[index] = idx + } + + // Append new identifiers to log. + entry := &LogEntry{ + Type: LogEntryTypeInsertColumn, + Index: []byte(index), + IDs: make([]uint64, 0, len(values)), + Keys: make([][]byte, 0, len(values)), + } + + check := make(map[string]uint64) + for i := range values { + if ret[i] != 0 { + continue + } + v, found := check[values[i]] + if !found { + idx.seq++ + v = idx.seq + check[values[i]] = v + } + + ret[i] = v + + entry.IDs = append(entry.IDs, v) + entry.Keys = append(entry.Keys, []byte(values[i])) + } + + // Write entry. + if err := s.appendEntry(entry); err != nil { + return nil, err + } + + return ret, nil +} + +// TranslateColumnToString converts a uint64 id to its associated string value. +// If the id is not associated with a string value then a blank string is returned. +func (s *TranslateFile) TranslateColumnToString(index string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + if ret, ok := idx.keyByID(value); ok { + s.mu.RUnlock() + return string(ret), nil + } + } + s.mu.RUnlock() + return "", nil +} + +func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + key := frameKey{index, frame} + + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.rows[key]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // Return error if not all values could be translated and this store is read-only. + if s.IsReadOnly() { + return ret, ErrTranslateStoreReadOnly + } + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.rows[key] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create map if it doesn't exists. + if idx == nil { + idx = newIndex(s.data) + s.rows[key] = idx + } + + // Append new identifiers to log. + entry := &LogEntry{ + Type: LogEntryTypeInsertRow, + Index: []byte(index), + Frame: []byte(frame), + IDs: make([]uint64, 0, len(values)), + Keys: make([][]byte, 0, len(values)), + } + check := make(map[string]uint64) + for i := range values { + if ret[i] != 0 { + continue + } + + v, found := check[values[i]] + if !found { + idx.seq++ + v = idx.seq + check[values[i]] = v + } + ret[i] = v + entry.IDs = append(entry.IDs, v) + entry.Keys = append(entry.Keys, []byte(values[i])) + } + + // Write entry. + if err := s.appendEntry(entry); err != nil { + return nil, err + } + + return ret, nil +} + +func (s *TranslateFile) TranslateRowToString(index, frame string, id uint64) (string, error) { + s.mu.RLock() + if idx := s.rows[frameKey{index, frame}]; idx != nil { + if ret, ok := idx.keyByID(id); ok { + s.mu.RUnlock() + return string(ret), nil + } + } + s.mu.RUnlock() + return "", nil +} + +// Reader returns a reader that streams the underlying data file. +func (s *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { + rc := NewTranslateFileReader(ctx, s, offset) + if err := rc.Open(); err != nil { + return nil, err + } + return rc, nil +} + +type LogEntry struct { + Type uint8 + Index []byte + Frame []byte + + IDs []uint64 + Keys [][]byte + + // Length of the entry, in bytes. + // This is only populated after ReadFrom() or WriteTo(). + Length uint64 +} + +// HeaderSize returns the number of bytes required for size, type, index, frame, & pair count. +func (e *LogEntry) HeaderSize() int64 { + sz := UvarintSize(e.Length) + // total entry length + 1 + // type + UvarintSize(uint64(len(e.Index))) + len(e.Index) + // Index length and data + UvarintSize(uint64(len(e.Frame))) + len(e.Frame) + // Frame length and data + UvarintSize(uint64(len(e.IDs))) // ID/Key pair count + return int64(sz) +} + +// ReadFrom deserializes a LogEntry from r. r must be a ByteReader. +func (e *LogEntry) ReadFrom(r io.Reader) (_ int64, err error) { + br := r.(io.ByteReader) + + // Read the entry length. + if e.Length, err = binary.ReadUvarint(br); err != nil { + return int64(UvarintSize(e.Length)), err + } + + // Slurp entire entry and replace reader. + buf := make([]byte, e.Length) + n, err := io.ReadFull(r, buf) + n64 := int64(n + UvarintSize(e.Length)) + if err != nil { + return n64, err + } + bufr := bytes.NewReader(buf) + br, r = bufr, bufr + + // Read the entry type. + if err := binary.Read(r, binary.BigEndian, &e.Type); err != nil { + return n64, err + } + + // Read index name. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz == 0 { + e.Index = nil + } else { + e.Index = make([]byte, sz) + if _, err := io.ReadFull(r, e.Index); err != nil { + return n64, err + } + } + + // Read frame name. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz == 0 { + e.Frame = nil + } else { + e.Frame = make([]byte, sz) + if _, err := io.ReadFull(r, e.Frame); err != nil { + return n64, err + } + } + + // Read key count. + if n, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if n == 0 { + e.IDs, e.Keys = nil, nil + } else { + e.IDs, e.Keys = make([]uint64, n), make([][]byte, n) + } + + // Read each id/key pairs. + for i := range e.Keys { + // Read identifier. + if e.IDs[i], err = binary.ReadUvarint(br); err != nil { + return n64, err + } + + // Read key. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz > 0 { + e.Keys[i] = make([]byte, sz) + if _, err := io.ReadFull(r, e.Keys[i]); err != nil { + return n64, err + } + } + } + return n64, nil +} + +// WriteTo serializes a LogEntry to w. +func (e *LogEntry) WriteTo(w io.Writer) (_ int64, err error) { + var buf bytes.Buffer + b := make([]byte, binary.MaxVarintLen64) + + // Write the entry type. + if err := binary.Write(&buf, binary.BigEndian, e.Type); err != nil { + return 0, err + } + + // Write the index name. + sz := binary.PutUvarint(b, uint64(len(e.Index))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Index); err != nil { + return 0, err + } + + // Write frame name. + sz = binary.PutUvarint(b, uint64(len(e.Frame))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Frame); err != nil { + return 0, err + } + + // Write key count. + sz = binary.PutUvarint(b, uint64(len(e.IDs))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } + + // Write each id/key pairs. + for i := range e.Keys { + // Write identifier. + sz = binary.PutUvarint(b, e.IDs[i]) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } + + // Write key. + sz = binary.PutUvarint(b, uint64(len(e.Keys[i]))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Keys[i]); err != nil { + return 0, err + } + } + + // Write buffer size. + e.Length = uint64(buf.Len()) + sz = binary.PutUvarint(b, e.Length) + if n, err := w.Write(b[:sz]); err != nil { + return int64(n), err + } + + // Write buffer. + n, err := buf.WriteTo(w) + return int64(sz) + n, err +} + +// ValidLogEntriesLen returns the maximum length of p that contains valid entries. +func ValidLogEntriesLen(p []byte) (n int) { + r := bytes.NewReader(p) + for { + if sz, err := binary.ReadUvarint(r); err != nil { + return n + } else if off, err := r.Seek(int64(sz), io.SeekCurrent); err != nil { + return n + } else if off > int64(len(p)) { + return n + } else { + n = int(off) + } + } +} + +type frameKey struct { + index string + frame string +} + +const defaultLoadFactor = 90 + +// index represents a two-way index between IDs and keys. +type index struct { + seq uint64 // autoincrement sequence + data []byte // memory-mapped file containing key data + + // RHH hashmap for id-to-offset mapping. + // This is required so we don't need to store key data on the heap. + // https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf + elems []elem // id/offset key pairs + n uint64 // number of inuse elements + mask uint64 // mask applied for modulus + threshold uint64 // threshold when capacity doubles + loadFactor int // factor used to calculate threshold + + // Builtin hashmap for offset-to-id mapping. + offsetsByID map[uint64]int64 +} + +func newIndex(data []byte) *index { + idx := &index{ + data: data, + offsetsByID: make(map[uint64]int64), + + loadFactor: defaultLoadFactor, + } + idx.alloc(pow2(uint64(256))) + return idx +} + +// keyByID returns the key for a given ID, if it exists. +func (idx *index) keyByID(id uint64) ([]byte, bool) { + offset, ok := idx.offsetsByID[id] + if !ok { + return nil, false + } + return idx.lookupKey(offset), true +} + +// idByKey returns the ID for a given key, if it exists. +func (idx *index) idByKey(key []byte) (uint64, bool) { + hash := hashKey(key) + pos := hash & idx.mask + + var dist uint64 + for { + if e := &idx.elems[pos]; e.hash == 0 { + return 0, false + } else if dist > idx.dist(e.hash, pos) { + return 0, false + } else if e.hash == hash && bytes.Equal(idx.lookupKey(e.offset), key) { + return e.id, true + } + + pos = (pos + 1) & idx.mask + dist++ + } +} + +// insert adds the id/offset pair to the index. +// This function will resize the map if it crosses the threshold. +func (idx *index) insert(id uint64, offset int64) { + idx.n++ + + // Add to reverse lookup. + idx.offsetsByID[id] = offset + + // Grow the map if we've run out of slots. + if idx.n > idx.threshold { + elems, capacity := idx.elems, uint64(len(idx.elems)) + idx.alloc(uint64(len(idx.elems) * 2)) + + for i := uint64(0); i < capacity; i++ { + e := &elems[i] + if e.hash == 0 { + continue + } + idx.insertIDbyOffset(e.offset, e.id) + } + } + + // If the key was overwritten then decrement the size. + if overwritten := idx.insertIDbyOffset(offset, id); overwritten { + idx.n-- + } +} + +// insertIDbyOffset writes to the RHH id-by-offset map. +func (idx *index) insertIDbyOffset(offset int64, id uint64) (overwritten bool) { + key := idx.lookupKey(offset) + hash := hashKey(key) + pos := hash & idx.mask + + var dist uint64 + for { + e := &idx.elems[pos] + + // Exit if a matching or empty slot exists. + if e.hash == 0 { + e.hash, e.offset, e.id = hash, offset, id + return false + } else if bytes.Equal(idx.lookupKey(e.offset), key) { + e.hash, e.offset, e.id = hash, offset, id + return true + } + + // Swap if current element has a lower probe distance. + d := idx.dist(e.hash, pos) + if d < dist { + hash, e.hash = e.hash, hash + offset, e.offset = e.offset, offset + id, e.id = e.id, id + dist = d + } + + // Move position forward. + pos = (pos + 1) & idx.mask + dist++ + } +} + +// lookupKey returns the key at the given offset in the memory-mapped file. +func (idx *index) lookupKey(offset int64) []byte { + data := idx.data[offset:] + n, sz := binary.Uvarint(data) + if sz == 0 { + return nil + } + return data[sz : sz+int(n)] +} + +func (idx *index) alloc(capacity uint64) { + idx.elems = make([]elem, capacity) + idx.threshold = (capacity * uint64(idx.loadFactor)) / 100 + idx.mask = uint64(capacity - 1) +} + +func (idx *index) dist(hash, i uint64) uint64 { + return (i + uint64(len(idx.elems)) - (hash & idx.mask)) & idx.mask +} + +type elem struct { + offset int64 + id uint64 + hash uint64 +} + +func (e *elem) reset() { + e.offset = 0 + e.id = 0 + e.hash = 0 +} + +func hashKey(key []byte) uint64 { + h := xxhash.Sum64(key) + if h == 0 { + h = 1 + } + return h +} + +func pow2(v uint64) uint64 { + for i := uint64(2); i < 1<<62; i *= 2 { + if i >= v { + return i + } + } + panic("unreachable") +} + +// TranslateFileReader implements a reader that continuously streams data from a store. +type TranslateFileReader struct { + ctx context.Context + store *TranslateFile + file *os.File + offset int64 + notify <-chan struct{} + + once sync.Once + closing chan struct{} +} + +// NewTranslateFileReader returns a new instance of TranslateFileReader. +func NewTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *TranslateFileReader { + return &TranslateFileReader{ + ctx: ctx, + store: store, + offset: offset, + notify: store.WriteNotify(), + closing: make(chan struct{}), + } +} + +// Open initializes the reader. +func (r *TranslateFileReader) Open() (err error) { + if r.file, err = os.Open(r.store.Path); err != nil { + return err + } + return nil +} + +// Close closes the underlying file reader. +func (r *TranslateFileReader) Close() error { + r.once.Do(func() { close(r.closing) }) + + if r.file != nil { + return r.file.Close() + } + return nil +} + +// Read reads the next section of the available data to p. This should always +// read from the start of an entry and read n bytes to the end of another entry. +func (r *TranslateFileReader) Read(p []byte) (n int, err error) { + for { + // Obtain notification channel before we check for new data. + notify := r.store.WriteNotify() + + // Exit if we can read one or more valid entries or we receive an error. + if n, err = r.read(p); n > 0 || err != nil { + return n, err + } + + // Wait for new data or close. + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + case <-r.closing: + return 0, ErrTranslateStoreReaderClosed + case <-r.store.Closing(): + return 0, ErrTranslateStoreClosed + case <-notify: + continue + } + } +} + +// read writes the bytes for zero or more valid entries to p. +func (r *TranslateFileReader) read(p []byte) (n int, err error) { + sz := r.store.Size() + + // Exit if there is no new data. + if sz < r.offset { + return 0, fmt.Errorf("pilosa: translate store reader past file size: sz=%d off=%d", sz, r.offset) + } else if sz == r.offset { + return 0, nil + } + + // Shorten buffer to maximum read size. + if max := sz - r.offset; int64(len(p)) > max { + p = p[:max] + } + + // Read data from file at offset. + // Limit the number of bytes read to only whole entries. + n, err = r.file.ReadAt(p, r.offset) + n = ValidLogEntriesLen(p[:n]) + r.offset += int64(n) + return n, err +} + +// Copied & modified from encoding/binary. +func UvarintSize(x uint64) (i int) { + for x >= 0x80 { + x >>= 7 + i++ + } + return i + 1 +} + +func hexdump(b []byte) { os.Stderr.Write([]byte(hex.Dump(b))) } diff --git a/translate_test.go b/translate_test.go new file mode 100644 index 000000000..c73853d69 --- /dev/null +++ b/translate_test.go @@ -0,0 +1,565 @@ +package pilosa_test + +import ( + "bufio" + "context" + "fmt" + "io/ioutil" + "math/rand" + "os" + "reflect" + "strconv" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/pilosa/pilosa" +) + +func TestTranslateFile_TranslateColumn(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return "". + if value, err := s.TranslateColumnToString("IDX0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + // Reopen the store. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + + // Ensure translation is still correct after reopen. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure translation is still correct after reopen. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } +} + +func TestTranslateFile_TranslateColumn_Large(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate key/values. + for i := 0; i < 1000000; i += 1000 { + keys := make([]string, 1000) + for j := 0; j < 1000; j++ { + keys[j] = strconv.Itoa(i + j + 1) + } + + ids, err := s.TranslateColumnsToUint64("IDX0", keys) + if err != nil { + t.Fatal(err) + } + + for j, id := range ids { + if exp := uint64(i + j + 1); id != exp { + t.Fatalf("unexpected id: got=%d, exp=%d", id, exp) + } + } + } + + // Verify values can be returned. + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } + + // Reopen and re-verify. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } +} + +func TestTranslateFile_TranslateRow(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return blank. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + // Reopen the store. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Translate new row and increment sequence. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } +} + +func TestTranslateFile_TranslateRow_Large(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate key/values. + for i := 0; i < 1000000; i += 1000 { + keys := make([]string, 1000) + for j := 0; j < 1000; j++ { + keys[j] = strconv.Itoa(i + j + 1) + } + + ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", keys) + if err != nil { + t.Fatal(err) + } + + for j, id := range ids { + if exp := uint64(i + j + 1); id != exp { + t.Fatalf("unexpected id: got=%d, exp=%d", id, exp) + } + } + } + + // Verify values can be returned. + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } + + // Reopen and re-verify. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } +} + +func TestTranslateFile_Reader(t *testing.T) { + t.Run("NoOffset", func(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + rc, err := s.Reader(context.Background(), 0) + if err != nil { + t.Fatal(err) + } + brc := bufio.NewReader(rc) + defer rc.Close() + + // Read first entry. Should read 'entry length' (13) plus uvarint(size) (1) = 14b. + var entry pilosa.LogEntry + if n, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if n != 14 { + t.Fatalf("unexpected n: %d", n) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertColumn, + Index: []byte("IDX0"), + IDs: []uint64{1}, + Keys: [][]byte{[]byte("foo")}, + Length: 13, + }); diff != "" { + t.Fatal(diff) + } + + // Read second entry. + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertRow, + Index: []byte("IDX0"), + Frame: []byte("FRAME0"), + IDs: []uint64{1, 2}, + Keys: [][]byte{[]byte("bar"), []byte("baz")}, + Length: 24, + }); diff != "" { + t.Fatal(diff) + } + + // Write new entry. + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"xyz"}); err != nil { + t.Fatal(err) + } + + // Read new entry. + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertColumn, + Index: []byte("IDX0"), + IDs: []uint64{2}, + Keys: [][]byte{[]byte("xyz")}, + Length: 13, + }); diff != "" { + t.Fatal(diff) + } + + // Close reader and ensure it returns EOF. + if err := rc.Close(); err != nil { + t.Fatal(err) + } else if _, err := entry.ReadFrom(brc); err != pilosa.ErrTranslateStoreReaderClosed { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("WithOffset", func(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + // Start offset after the first entry. + rc, err := s.Reader(context.Background(), 14) + if err != nil { + t.Fatal(err) + } + brc := bufio.NewReader(rc) + defer rc.Close() + + // This should be the second entry. + var entry pilosa.LogEntry + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertRow, + Index: []byte("IDX0"), + Frame: []byte("FRAME0"), + IDs: []uint64{1, 2}, + Keys: [][]byte{[]byte("bar"), []byte("baz")}, + Length: 24, + }); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestTranslateFile_PrimaryTranslateStore(t *testing.T) { + // Create a primary store that accepts writes. + primary := MustOpenTranslateFile() + defer primary.MustClose() + + // Create a replica that accepts writes from primary. + replica := NewTranslateFile() + replica.PrimaryTranslateStore = primary + if err := replica.Open(); err != nil { + t.Fatal(err) + } + defer replica.MustClose() + + // Write to the primary. + if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until writes appear. + if err := retryFor(2*time.Second, func() error { + // Verify that replica have received writes. + if value, err := replica.TranslateColumnToString("IDX0", 1); err != nil { + return err + } else if value != "foo" { + return fmt.Errorf("unexpected column 1 value: %s", value) + } + + if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 1); err != nil { + return err + } else if value != "bar" { + return fmt.Errorf("unexpected row 1 value: %s", value) + } + + if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + return err + } else if value != "baz" { + return fmt.Errorf("unexpected row 2 value: %s", value) + } + + return nil + }); err != nil { + t.Fatal(err) + } + + // Disconnect primary store & write more values. + if err := primary.Reopen(); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until write appear. + if err := retryFor(2*time.Second, func() error { + if value, err := replica.TranslateColumnToString("IDX0", 2); err != nil { + return err + } else if value != "baz" { + return fmt.Errorf("unexpected column 2 value: %s", value) + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Disconnect replica store & write more values. + if err := replica.Reopen(); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foobar"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until write appear. + if err := retryFor(2*time.Second, func() error { + if value, err := replica.TranslateColumnToString("IDX0", 3); err != nil { + return err + } else if value != "foobar" { + return fmt.Errorf("unexpected column 3 value: %s", value) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func BenchmarkTranslateFile_TranslateColumnsToUint64(b *testing.B) { + const batchSize = 1000 + + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate keys before benchmark begins + keySets := make([][]string, b.N/batchSize) + for i := range keySets { + keySets[i] = make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i) + } + } + + b.ResetTimer() + + for _, keySet := range keySets { + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkTranslateFile_TranslateColumnToString(b *testing.B) { + const batchSize = 1000 + + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate keys before benchmark begins + for i := 0; i < b.N; i += batchSize { + keySet := make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySet[j] = fmt.Sprintf("%08d%08d", jv, i) + } + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } + + // Generate random key access. + perm := rand.New(rand.NewSource(0)).Perm(b.N) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil { + b.Fatal(err) + } + } +} + +type TranslateFile struct { + *pilosa.TranslateFile +} + +func NewTranslateFile() *TranslateFile { + f, err := ioutil.TempFile("", "") + if err != nil { + panic(err) + } + f.Close() + + s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile()} + s.Path = f.Name() + return s +} + +func MustOpenTranslateFile() *TranslateFile { + s := NewTranslateFile() + if err := s.Open(); err != nil { + panic(err) + } + return s +} + +func (s *TranslateFile) Close() error { + defer os.Remove(s.Path) + return s.TranslateFile.Close() +} + +func (s *TranslateFile) MustClose() { + if err := s.Close(); err != nil { + panic(err) + } +} + +// Reopen closes the store and opens a new instance of it for the same path. +func (s *TranslateFile) Reopen() error { + prev := s.TranslateFile + if err := s.TranslateFile.Close(); err != nil { + return err + } + + s.TranslateFile = pilosa.NewTranslateFile() + s.Path = prev.Path + s.PrimaryTranslateStore = prev.PrimaryTranslateStore + if err := s.Open(); err != nil { + return err + } + return nil +} + +// retryFor executes fn every 100ms until d time passes or until fn return nil. +func retryFor(d time.Duration, fn func() error) (err error) { + timer, ticker := time.NewTimer(d), time.NewTicker(100*time.Millisecond) + defer timer.Stop() + defer ticker.Stop() + + for { + if err = fn(); err == nil { + return nil + } + + select { + case <-timer.C: + return err + case <-ticker.C: + } + } +} From 9a74763156e7a21eb3931f1c7dd3e71cd2a1c8d6 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 18 Jun 2018 12:40:59 -0500 Subject: [PATCH 73/78] move pilosa/test/client.go helpers into pilosa/client_test.go --- http/client_test.go | 28 +++++++++++++++++++++------- test/client.go | 35 ----------------------------------- 2 files changed, 21 insertions(+), 42 deletions(-) delete mode 100644 test/client.go diff --git a/http/client_test.go b/http/client_test.go index 851f1140e..185a3c378 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -148,10 +148,10 @@ func TestClient_MultiNode(t *testing.T) { hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache() // Connect to each node to compare results. - client := make([]*test.Client, 3) - client[0] = test.MustNewClient(s[0].Host(), defaultClient) - client[1] = test.MustNewClient(s[1].Host(), defaultClient) - client[2] = test.MustNewClient(s[2].Host(), defaultClient) + client := make([]*Client, 3) + client[0] = MustNewClient(s[0].Host(), defaultClient) + client[1] = MustNewClient(s[1].Host(), defaultClient) + client[2] = MustNewClient(s[2].Host(), defaultClient) topN := 4 queryRequest := &internal.QueryRequest{ @@ -231,7 +231,7 @@ func TestClient_Import(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host(), defaultClient) + c := MustNewClient(s.Host(), defaultClient) if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, @@ -276,7 +276,7 @@ func TestClient_ImportValue(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host(), defaultClient) + c := MustNewClient(s.Host(), defaultClient) if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{ {ColumnID: 1, Value: -10}, {ColumnID: 2, Value: 20}, @@ -345,7 +345,7 @@ func TestClient_FragmentBlocks(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Retrieve blocks. - c := test.MustNewClient(s.Host(), defaultClient) + c := MustNewClient(s.Host(), defaultClient) blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0) if err != nil { t.Fatal(err) @@ -362,3 +362,17 @@ func TestClient_FragmentBlocks(t *testing.T) { t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks)) } } + +// Client represents a test wrapper for pilosa.Client. +type Client struct { + *http.InternalClient +} + +// MustNewClient returns a new instance of Client. Panic on error. +func MustNewClient(host string, h *gohttp.Client) *Client { + c, err := http.NewInternalClient(host, h) + if err != nil { + panic(err) + } + return &Client{InternalClient: c} +} diff --git a/test/client.go b/test/client.go deleted file mode 100644 index 34d5e17d0..000000000 --- a/test/client.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - gohttp "net/http" - - "github.com/pilosa/pilosa/http" -) - -// Client represents a test wrapper for pilosa.Client. -type Client struct { - *http.InternalClient -} - -// MustNewClient returns a new instance of Client. Panic on error. -func MustNewClient(host string, h *gohttp.Client) *Client { - c, err := http.NewInternalClient(host, h) - if err != nil { - panic(err) - } - return &Client{InternalClient: c} -} From 60dee04ed130c06e482471a11c06016e22072818 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 18 Jun 2018 12:52:58 -0500 Subject: [PATCH 74/78] move pilosa/test/attr.go into pilosa/attr_test.go --- attr_test.go | 79 ++++++++++++++++++++++++++++++++++++++++++--- test/attr.go | 90 ---------------------------------------------------- 2 files changed, 74 insertions(+), 95 deletions(-) delete mode 100644 test/attr.go diff --git a/attr_test.go b/attr_test.go index 8253c8e1d..0c848e2ff 100644 --- a/attr_test.go +++ b/attr_test.go @@ -15,15 +15,20 @@ package pilosa_test import ( + "io/ioutil" + "os" "reflect" + "runtime" + "sync" "testing" - "github.com/pilosa/pilosa/test" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" ) // Ensure database can set and retrieve column attributes. func TestAttrStore_Attrs(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -52,7 +57,7 @@ func TestAttrStore_Attrs(t *testing.T) { // Ensure database returns a non-nil empty map if unset. func TestAttrStore_Attrs_Empty(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() if m, err := s.Attrs(100); err != nil { @@ -64,7 +69,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) { // Ensure database can unset attributes if explicitly set to nil. func TestAttrStore_Attrs_Unset(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -84,7 +89,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) { // Ensure attribute block checksums can be returned. func TestAttrStore_Blocks(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -123,3 +128,67 @@ func TestAttrStore_Blocks(t *testing.T) { t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2]) } } + +// AttrStore represents a test wrapper for pilosa.AttrStore. +type AttrStore struct { + pilosa.AttrStore +} + +// NewAttrStore returns a new instance of AttrStore. +func NewAttrStore(string) pilosa.AttrStore { + f, err := ioutil.TempFile("", "pilosa-attr-") + if err != nil { + panic(err) + } + f.Close() + os.Remove(f.Name()) + + return &AttrStore{boltdb.NewAttrStore(f.Name())} +} + +func BenchmarkAttrStore_Duplicate(b *testing.B) { + s := MustOpenAttrStore() + defer s.Close() + + // Set attributes. + const n = 5 + for i := 0; i < n; i++ { + if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + // Update attributes with an existing subset. + cpuN := runtime.GOMAXPROCS(0) + var wg sync.WaitGroup + for i := 0; i < cpuN; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < b.N/cpuN; j++ { + if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { + b.Fatal(err) + } + } + }() + } + wg.Wait() +} + +// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. +func MustOpenAttrStore() pilosa.AttrStore { + s := NewAttrStore("") + if err := s.Open(); err != nil { + panic(err) + } + return s +} + +// Close closes the database and removes the underlying data. +func (s *AttrStore) Close() error { + defer os.RemoveAll(s.Path()) + return s.AttrStore.Close() +} diff --git a/test/attr.go b/test/attr.go deleted file mode 100644 index 16e8ff334..000000000 --- a/test/attr.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "io/ioutil" - "os" - "runtime" - "sync" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/boltdb" -) - -// AttrStore represents a test wrapper for pilosa.AttrStore. -type AttrStore struct { - pilosa.AttrStore -} - -// NewAttrStore returns a new instance of AttrStore. -func NewAttrStore(string) pilosa.AttrStore { - f, err := ioutil.TempFile("", "pilosa-attr-") - if err != nil { - panic(err) - } - f.Close() - os.Remove(f.Name()) - - return &AttrStore{boltdb.NewAttrStore(f.Name())} -} - -func BenchmarkAttrStore_Duplicate(b *testing.B) { - s := MustOpenAttrStore() - defer s.Close() - - // Set attributes. - const n = 5 - for i := 0; i < n; i++ { - if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { - b.Fatal(err) - } - } - - b.ReportAllocs() - b.ResetTimer() - - // Update attributes with an existing subset. - cpuN := runtime.GOMAXPROCS(0) - var wg sync.WaitGroup - for i := 0; i < cpuN; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < b.N/cpuN; j++ { - if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { - b.Fatal(err) - } - } - }() - } - wg.Wait() -} - -// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. -func MustOpenAttrStore() pilosa.AttrStore { - s := NewAttrStore("") - if err := s.Open(); err != nil { - panic(err) - } - return s -} - -// Close closes the database and removes the underlying data. -func (s *AttrStore) Close() error { - defer os.RemoveAll(s.Path()) - return s.AttrStore.Close() -} From 277ee1e25e8ef02163bc1bd9d4d2f665512e71c6 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 18 Jun 2018 13:28:24 -0500 Subject: [PATCH 75/78] added fields meta to index http endpoint --- http/handler.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/http/handler.go b/http/handler.go index df5108826..1d43fea5d 100644 --- a/http/handler.go +++ b/http/handler.go @@ -343,16 +343,22 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusNotFound) return } + fields := make(map[string]string) + for _, field := range index.Fields() { + fields["name"] = field.Name() + } if err := json.NewEncoder(w).Encode(getIndexResponse{ - map[string]string{"name": index.Name()}, + Index: map[string]string{"name": index.Name()}, + Fields: fields, }); err != nil { h.Logger.Printf("write response error: %s", err) } } type getIndexResponse struct { - Index map[string]string `json:"index"` + Index map[string]string `json:"index"` + Fields map[string]string `json:"fields"` } type postIndexRequest struct { From 07ab60d6cf76f3e77aa7a206996b57b6c478ef03 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 18 Jun 2018 13:58:12 -0500 Subject: [PATCH 76/78] adjust requirements to match schema response --- http/handler.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/http/handler.go b/http/handler.go index 1d43fea5d..c509971a2 100644 --- a/http/handler.go +++ b/http/handler.go @@ -337,29 +337,31 @@ func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { // handleGetIndex handles GET /index/ requests. func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] - index, err := h.API.Index(r.Context(), indexName) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) + var info *pilosa.IndexInfo + for _, idx := range h.API.Schema(r.Context()) { + if strings.Compare(idx.Name, indexName) == 0 { + info = idx + break + } + } + if info == nil { + http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound) return } - fields := make(map[string]string) - for _, field := range index.Fields() { - fields["name"] = field.Name() - } - if err := json.NewEncoder(w).Encode(getIndexResponse{ - Index: map[string]string{"name": index.Name()}, - Fields: fields, - }); err != nil { + if err := json.NewEncoder(w).Encode(info); err != nil { h.Logger.Printf("write response error: %s", err) } } +/* type getIndexResponse struct { Index map[string]string `json:"index"` Fields map[string]string `json:"fields"` } +*/ type postIndexRequest struct { Options pilosa.IndexOptions `json:"options"` From 18c67269ae794490653604b298a1f1fcd8812a39 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 18 Jun 2018 14:08:22 -0500 Subject: [PATCH 77/78] clarity --- http/handler.go | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/http/handler.go b/http/handler.go index c509971a2..9aea6e12c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -337,32 +337,18 @@ func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { // handleGetIndex handles GET /index/ requests. func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - var info *pilosa.IndexInfo for _, idx := range h.API.Schema(r.Context()) { - if strings.Compare(idx.Name, indexName) == 0 { - info = idx - break + if idx.Name == indexName { + if err := json.NewEncoder(w).Encode(idx); err != nil { + h.Logger.Printf("write response error: %s", err) + } + return } } - if info == nil { - http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound) - return - } - - if err := json.NewEncoder(w).Encode(info); err != nil { - h.Logger.Printf("write response error: %s", err) - } + http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound) } -/* -type getIndexResponse struct { - Index map[string]string `json:"index"` - Fields map[string]string `json:"fields"` -} -*/ - type postIndexRequest struct { Options pilosa.IndexOptions `json:"options"` } From 56ed9bfbe1cf50810b4b13d28662f3c5e401698f Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 18 Jun 2018 18:47:29 -0500 Subject: [PATCH 78/78] remove broadcaster methods from gossip- don't use sendAsync anywhere --- cluster.go | 2 +- gossip/gossip.go | 46 ---------------------------------------------- server.go | 3 +-- server/server.go | 2 -- view.go | 4 ++-- 5 files changed, 4 insertions(+), 53 deletions(-) diff --git a/cluster.go b/cluster.go index 4ddaf3a87..8dc3a60d1 100644 --- a/cluster.go +++ b/cluster.go @@ -910,7 +910,7 @@ func (c *Cluster) open() error { Event: uint32(NodeJoin), Node: EncodeNode(c.Node), } - if err := c.Broadcaster.SendAsync(msg); err != nil { + if err := c.Broadcaster.SendSync(msg); err != nil { return fmt.Errorf("sending restart NodeJoin: %v", err) } diff --git a/gossip/gossip.go b/gossip/gossip.go index dfcb2f759..9154e9fc7 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -24,8 +24,6 @@ import ( "sync" "time" - "golang.org/x/sync/errgroup" - "github.com/gogo/protobuf/proto" "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" @@ -36,7 +34,6 @@ import ( // Ensure GossipMemberSet implements interfaces. var _ pilosa.BroadcastReceiver = &GossipMemberSet{} -var _ pilosa.Gossiper = &GossipMemberSet{} var _ memberlist.Delegate = &GossipMemberSet{} // GossipMemberSet represents a gossip implementation of MemberSet using memberlist. @@ -240,49 +237,6 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe return g, nil } -// SendSync implementation of the Broadcaster interface. -func (g *GossipMemberSet) SendSync(pb proto.Message) error { - msg, err := pilosa.MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshal message: %s", err) - } - - mlist := g.memberlist - - // Direct sends the message directly to every node. - // An error from any node raises an error on the entire operation. - // - // Gossip uses the gossip protocol to eventually deliver the message - // to every node. - var eg errgroup.Group - for _, n := range mlist.Members() { - // Don't send the message to the local node. - if n == mlist.LocalNode() { - continue - } - node := n - eg.Go(func() error { - return mlist.SendToTCP(node, msg) - }) - } - return eg.Wait() -} - -// SendAsync implementation of the Gossiper interface. -func (g *GossipMemberSet) SendAsync(pb proto.Message) error { - msg, err := pilosa.MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshal message: %s", err) - } - - b := &broadcast{ - msg: msg, - notify: nil, - } - g.broadcasts.QueueBroadcast(b) - return nil -} - // NodeMeta implementation of the memberlist.Delegate interface. func (g *GossipMemberSet) NodeMeta(limit int) []byte { buf, err := proto.Marshal(pilosa.EncodeNode(g.node)) diff --git a/server.go b/server.go index 4a453cef5..dd26a2c0b 100644 --- a/server.go +++ b/server.go @@ -62,7 +62,6 @@ type Server struct { handler Handler Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver - Gossiper Gossiper systemInfo SystemInfo gcNotifier GCNotifier NewAttrStore func(string) AttrStore @@ -547,7 +546,7 @@ func (s *Server) SendSync(pb proto.Message) error { // SendAsync represents an implementation of Broadcaster. func (s *Server) SendAsync(pb proto.Message) error { - return s.Gossiper.SendAsync(pb) + return ErrNotImplemented } // SendTo represents an implementation of Broadcaster. diff --git a/server/server.go b/server/server.go index 0c4184d60..b6bf6e864 100644 --- a/server/server.go +++ b/server/server.go @@ -268,7 +268,6 @@ func (m *Command) SetupNetworking() error { m.Server.Broadcaster = pilosa.NopBroadcaster m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet(m.Server.Cluster.Nodes) m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver - m.Server.Gossiper = pilosa.NopGossiper return nil } @@ -313,7 +312,6 @@ func (m *Command) SetupNetworking() error { m.Server.Cluster.MemberSet = gossipMemberSet m.Server.Broadcaster = m.Server m.Server.BroadcastReceiver = gossipMemberSet - m.Server.Gossiper = gossipMemberSet return nil } diff --git a/view.go b/view.go index b21c3f15a..428fc6f54 100644 --- a/view.go +++ b/view.go @@ -237,13 +237,13 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { v.maxSlice = slice // Send the create slice message to all nodes. - err := v.broadcaster.SendAsync( + err := v.broadcaster.SendSync( &internal.CreateSliceMessage{ Index: v.index, Slice: slice, }) if err != nil { - return nil, errors.Wrap(err, "sending message") + return nil, errors.Wrap(err, "sending createslice message") } }