From 5b43a84a3be25519cc2ce0fb2f68c6f46552aa42 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Feb 2021 17:09:50 -0600 Subject: [PATCH 01/11] . --- cmd/sauron/design.org | 199 ++++ cmd/sauron/httputil.go | 274 +++++ cmd/sauron/main.go | 1802 +++++++++++++++++++++++++++++++++ cmd/sauron/populate.sh | 10 + cmd/sauron/sample_schema.json | 1 + cmd/sauron/sauron_test.go | 396 ++++++++ cmd/sauron/schema_test.go | 353 +++++++ cmd/sauron/sqlgen_test.go | 260 +++++ cmd/sauron/test_sqlite.sh | 51 + cmd/sauron/vprint.go | 167 +++ 10 files changed, 3513 insertions(+) create mode 100644 cmd/sauron/design.org create mode 100644 cmd/sauron/httputil.go create mode 100644 cmd/sauron/main.go create mode 100644 cmd/sauron/populate.sh create mode 100644 cmd/sauron/sample_schema.json create mode 100644 cmd/sauron/sauron_test.go create mode 100644 cmd/sauron/schema_test.go create mode 100644 cmd/sauron/sqlgen_test.go create mode 100755 cmd/sauron/test_sqlite.sh create mode 100644 cmd/sauron/vprint.go diff --git a/cmd/sauron/design.org b/cmd/sauron/design.org new file mode 100644 index 000000000..aa579ab34 --- /dev/null +++ b/cmd/sauron/design.org @@ -0,0 +1,199 @@ + +* Oracle Design +- setup server that intersepts PQL queries and proxies to live pilosa server +- implement all the pilosa functions using SQL, specifically the sqlite flavor. This will + provide validation against pilosa results +- provide timings for each query path, PQL and SQL and compare +- this is intended to be a correctness check and not a performance check. the + sql solution is likely to be pretty slow with realitively small numbers ~1millions + #+BEGIN_SRC + .─────. + ,' `. ┌────────────┐ ┌────────┐ +;pilosa json: │ Sauron │ ┌──▶│ pilosa │ +: client ;──────────▶ │ │───┤ └────────┘ + ╲ ╱ │ │ │ ┌────────┐ + `. ,' └────────────┘ └──▶│ sqlite │ + `───' └────────┘ + #+END_SRC +* Setup Sql (sqlite) +#+name: setup test +#+header: :results silent +#+header: :db poc.db + #+BEGIN_SRC sqlite + create table bits (field string,row string, column bigint); + create table bsi (field string,column bigint , val bigint); + CREATE VIEW columns as + select distinct column from bits + UNION + select distinct column from bsi; + + INSERT INTO bits VALUES( 'color','red',1); + INSERT INTO bits VALUES( 'color','red',2); + INSERT INTO bits VALUES( 'color','red',3); + INSERT INTO bits VALUES( 'color','red',4); + INSERT INTO bits VALUES( 'color','red',5); + INSERT INTO bits VALUES( 'color','green',2); + INSERT INTO bits VALUES( 'color','green',4); + INSERT INTO bits VALUES( 'color','green',6); + INSERT INTO bits VALUES( 'color','green',8); + INSERT INTO bits VALUES( 'color','yellow',1); + INSERT INTO bits VALUES( 'color','yellow',3); + INSERT INTO bits VALUES( 'color','yellow',5); + INSERT INTO bits VALUES( 'color','yellow',7); + INSERT INTO bits VALUES( 'color','orange',5); + INSERT INTO bits VALUES( 'color','orange',6); + INSERT INTO bits VALUES( 'color','orange',7); + INSERT INTO bits VALUES( 'color','orange',8); + + INSERT INTO bsi VALUES( 'luminosity',1, 1); + INSERT INTO bsi VALUES( 'luminosity',2, 2); + INSERT INTO bsi VALUES( 'luminosity',3, 4); + INSERT INTO bsi VALUES( 'luminosity',4, 5); + INSERT INTO bsi VALUES( 'luminosity',5, 4); + INSERT INTO bsi VALUES( 'luminosity',6, 3); + INSERT INTO bsi VALUES( 'luminosity',7, 2); + INSERT INTO bsi VALUES( 'luminosity',8, 1); + #+END_SRC + +* TODO PQL to SQL TODO [10/11] +- [X] Row(color=red) + #+begin_src go +(*pql.Query)(&pql.Query{ + Calls: ([]*pql.Call)([]*pql.Call{ + (*pql.Call)(&pql.Call{ + Name: (string)("Row"), + Args: (map[string]interface{})(map[string]interface{}{ + (string)("color"): (string)("red"), + }), + Children: ([]*pql.Call)(nil), + Type: (pql.CallType)(0), + Precomputed: (map[uint64]interface{})(nil), + }), + }), + callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), + conditional: ([]string)(nil), +}) + #+end_src + #+BEGIN_SRC sqlite + select column from bits where field="color" AND row="red" + #+END_SRC +- [X] Count(Row(color=red)) + #+begin_src go +(*pql.Query)(&pql.Query{ + Calls: ([]*pql.Call)([]*pql.Call{ + (*pql.Call)(&pql.Call{ + Name: (string)("Count"), + Args: (map[string]interface{})(nil), + Children: ([]*pql.Call)([]*pql.Call{ + (*pql.Call)(&pql.Call{ + Name: (string)("Row"), + Args: (map[string]interface{})(map[string]interface{}{ + (string)("color"): (string)("red"), + }), + Children: ([]*pql.Call)(nil), + Type: (pql.CallType)(0), + Precomputed: (map[uint64]interface{})(nil), + }), + }), + Type: (pql.CallType)(0), + Precomputed: (map[uint64]interface{})(nil), + }), + }), + callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), + conditional: ([]string)(nil), +}) + + #+end_src + #+BEGIN_SRC sql + select count(*) from( + select column from bits where field="color" AND row="red" + ) + #+END_SRC +- [X] Intersect(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from bits where field="color" AND row="red" + intersect + select column from bits where field="color" AND row="green" + #+END_SRC +- [X] Union(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from bits where field="color" AND row="red" + union + select column from bits where field="color" AND row="green" + #+END_SRC +- [X] Difference(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from bits where field="color" AND row="red" + except + select column from bits where field="color" AND row="green" + #+END_SRC +- [X] Not(Row(color=red)) + #+BEGIN_SRC sql + select column from columns + except + select column from bits where field="color" AND row="red" + #+END_SRC +- [X] Xor(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from ( + select column from bits where field="color" AND row="green" + union + select column from bits where field="color" AND row="red" + ) + except + select column from ( + select column from bits where field="color" AND row="green" + intersect + select column from bits where field="color" AND row="red" + ) + #+END_SRC +- [X] Distinct(field=luminosity) + #+BEGIN_SRC sql + select distinct val from bsi where field="lumin" + #+END_SRC +- [X] Distinct(Row(color="red"),field=luminosity) + #+BEGIN_SRC sql + select distinct val from bsi + where field="luminosity" ANDcolumn in ( + select column from bits where row="color" AND row="red" + ) + #+END_SRC +- [X] Distinct(Intersect(Row(color="red"),Row(color="green")),luminosity) + #+BEGIN_SRC sql + select distinct val from bsi + where column in ( + select column from bits where field="color" AND row="red" + intersect + select column from bits where field="color" AND row="green" + ) + AND field="luminosity" + #+END_SRC + +- [X] Count(Union(Intersect(Row(color=red),Row(color=green)),Intersect(Row(color=yellow),Row(color=orange)))) +#+begin_src sqlite +select count(*) from ( +select column from ( + select column from bits where field="color" AND row="green" + intersect + select column from bits where field="color" AND row="red" +) +union +select column from ( + select column from bits where field="color" AND row="yellow" + intersect + select column from bits where field="color" AND row="orange" +)) + +- [ ] Rows(color) +#+begin_src sqlite +select distinct row from bits where field="color" +#+end_src + +- [ ] GroupBy(Rows(color)) + #+BEGIN_SRC sql + select row,count(*) from bits where field="color" group by row; + #+END_SRC +- [ ] GroupBy(Rows(color), Rows(other), limit=7) + #+BEGIN_SRC sql + select row,count(*) from bits where field="color" group by row; + #+END_SRC diff --git a/cmd/sauron/httputil.go b/cmd/sauron/httputil.go new file mode 100644 index 000000000..7056d40e8 --- /dev/null +++ b/cmd/sauron/httputil.go @@ -0,0 +1,274 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "log" + "mime" + "net/http" + "strings" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/pql" + "github.com/pkg/errors" +) + +// validHeaderAcceptJSON returns false if one or more Accept +// headers are present, but none of them are "application/json" +// (or any matching wildcard). Otherwise returns true. +func validHeaderAcceptJSON(header http.Header) bool { + return validHeaderAcceptType(header, "application", "json") +} + +func validHeaderAcceptType(header http.Header, typ, subtyp string) bool { + if v, found := header["Accept"]; found { + for _, v := range v { + t, _, err := mime.ParseMediaType(v) + if err != nil { + switch err { + case mime.ErrInvalidMediaParameter: + // This is an optional feature, so we can keep going anyway. + default: + continue + } + } + spl := strings.SplitN(t, "/", 2) + if len(spl) < 2 { + continue + } + switch { + case spl[0] == typ && spl[1] == subtyp: + return true + case spl[0] == "*" && spl[1] == subtyp: + return true + case spl[0] == typ && spl[1] == "*": + return true + case spl[0] == "*" && spl[1] == "*": + return true + } + } + return false + } + return true +} + +// successResponse is a general success/error struct for http responses. +type successResponse struct { + //h *Handler + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + //Error *Error `json:"error,omitempty"` + Error error +} + +// check determines success or failure based on the error. +// It also returns the corresponding http status code. +func (r *successResponse) check(err error) (statusCode int) { + if err == nil { + r.Success = true + return 0 + } + + cause := errors.Cause(err) + + // Determine HTTP status code based on the error type. + switch cause.(type) { + case pilosa.BadRequestError: + statusCode = http.StatusBadRequest + case pilosa.ConflictError: + statusCode = http.StatusConflict + case pilosa.NotFoundError: + statusCode = http.StatusNotFound + default: + statusCode = http.StatusInternalServerError + } + + r.Success = false + r.Error = err // = &Error{Message: err.Error()} + + return statusCode +} + +// write sends a response to the http.ResponseWriter based on the success +// status and the error. +func (r *successResponse) write(w http.ResponseWriter, err error) { + // Apply the error and get the status code. + statusCode := r.check(err) + + // Marshal the json response. + msg, err := json.Marshal(r) + if err != nil { + http.Error(w, string(msg), http.StatusInternalServerError) + return + } + + // Write the response. + if statusCode == 0 { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write(msg) + if err != nil { + log.Printf("error writing response: %v", err) + return + } + _, err = w.Write([]byte("\n")) + if err != nil { + log.Printf("error writing newline after response: %v", err) + return + } + } else { + http.Error(w, string(msg), statusCode) + } +} + +type postFieldRequest struct { + Options fieldOptions `json:"options"` +} + +// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values, +// and used for input validation. +type fieldOptions struct { + Type string `json:"type,omitempty"` + CacheType *string `json:"cacheType,omitempty"` + CacheSize *uint32 `json:"cacheSize,omitempty"` + Min *pql.Decimal `json:"min,omitempty"` + Max *pql.Decimal `json:"max,omitempty"` + Scale *int64 `json:"scale,omitempty"` + TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` + Keys *bool `json:"keys,omitempty"` + NoStandardView bool `json:"noStandardView,omitempty"` + ForeignIndex *string `json:"foreignIndex,omitempty"` +} + +func (o *fieldOptions) ToPilosaFieldOptions() *pilosa.FieldOptions { + + fo := &pilosa.FieldOptions{ + Type: o.Type, + + /* + //Base int64 `json:"base,omitempty"` + //BitDepth uint `json:"bitDepth,omitempty"` + Min: *o.Min, + Max: *o.Max, + Scale: *o.Scale, + NoStandardView: o.NoStandardView, + CacheSize: *o.CacheSize, + CacheType: *o.CacheType, + TimeQuantum: *o.TimeQuantum, + ForeignIndex: *o.ForeignIndex, + */ + } + if o.Keys != nil { + fo.Keys = *o.Keys + } + return fo +} + +func (o *fieldOptions) validate() error { + // Pointers to default values. + defaultCacheType := pilosa.DefaultCacheType + defaultCacheSize := uint32(pilosa.DefaultCacheSize) + + switch o.Type { + case pilosa.FieldTypeSet, "": + // Because FieldTypeSet is the default, its arguments are + // not required. Instead, the defaults are applied whenever + // a value does not exist. + if o.Type == "" { + o.Type = pilosa.FieldTypeSet + } + if o.CacheType == nil { + o.CacheType = &defaultCacheType + } + if o.CacheSize == nil { + o.CacheSize = &defaultCacheSize + } + if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type set")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type set")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) + } + case pilosa.FieldTypeInt: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { + return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) + } + case pilosa.FieldTypeDecimal: + if o.Scale == nil { + return pilosa.NewBadRequestError(errors.New("decimal field requires a scale argument")) + } else if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { + return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) + } + case pilosa.FieldTypeTime: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time")) + } else if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type time")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type time")) + } else if o.TimeQuantum == nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time")) + } + case pilosa.FieldTypeMutex: + if o.CacheType == nil { + o.CacheType = &defaultCacheType + } + if o.CacheSize == nil { + o.CacheSize = &defaultCacheSize + } + if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type mutex")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) + } + case pilosa.FieldTypeBool: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) + } else if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) + } else if o.Keys != nil { + return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool")) + } else if o.ForeignIndex != nil { + return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key")) + } + default: + return errors.Errorf("invalid field type: %s", o.Type) + } + return nil +} diff --git a/cmd/sauron/main.go b/cmd/sauron/main.go new file mode 100644 index 000000000..497e5346b --- /dev/null +++ b/cmd/sauron/main.go @@ -0,0 +1,1802 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + //"io" + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "os" + "os/exec" + "sort" + "strconv" + "strings" + "time" + + "modernc.org/sqlite" + _ "modernc.org/sqlite" + + //"github.com/gorilla/handlers" + "github.com/gorilla/mux" + "github.com/pilosa/pilosa/v2" + + "github.com/pilosa/pilosa/v2/encoding/proto" + "github.com/pilosa/pilosa/v2/pql" + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pkg/errors" + "github.com/shurcooL/go-goon" +) + +func main() { + +} + +// Sauron sees everything. +// And he converts PQL into SQL. +// +// The PQL is applied to a Pilosa and the SQL +// is applied to a SQL database. Then the +// query results are compared as they are +// returned to the client. +// +type Sauron struct { + cfg *SauronConfig + + sql *PQLToSQL + url string // "https://..." + addr string // host:port + mux *http.ServeMux + srv *http.Server + cmd *exec.Cmd + + // target of proxy + pilosaHttpURL string + + Handler http.Handler +} + +type SauronConfig struct { + Bind string // host:port + BindGRPC string // host:port + GossipPort string // port + DataDir string + + DatabaseName string // for sql database +} + +// IndexInfo2 and FieldInfo2 let us look up +// the schema details quickly by using maps +// to index the fields in an index. +type IndexInfo2 struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options pilosa.IndexOptions `json:"options"` + Fields map[string]*FieldInfo2 `json:"fields"` + ShardWidth uint64 `json:"shardWidth"` +} + +type FieldInfo2 struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options pilosa.FieldOptions `json:"options"` + Views []*pilosa.ViewInfo `json:"views,omitempty"` + indexInfo *IndexInfo2 +} + +func (fi *FieldInfo2) GetIndexName() string { + return fi.indexInfo.Name +} + +func (fi *FieldInfo2) GetIndexCreatedAt() int64 { + return fi.indexInfo.CreatedAt +} + +func (fi *FieldInfo2) GetCreatedAt() int64 { + return fi.CreatedAt +} + +func (fi *FieldInfo2) GetName() string { + return fi.Name +} + +type PQLToSQL struct { + schema pilosa.Schema + DB *sql.DB + + dbprefix string // prefixed to all tables + + // map from indexName to *IndexInfo2 + i2f map[string]*IndexInfo2 + ShardWidth uint64 +} + +func NewPQLToSQL(dbprefix string) *PQLToSQL { + return &PQLToSQL{ + dbprefix: strings.ToLower(dbprefix), // some SQL are not case sensitive, just use lower case. + i2f: make(map[string]*IndexInfo2), + ShardWidth: 1048576, + } +} + +func (ps *PQLToSQL) Start() error { + db, err := sql.Open("sqlite", fmt.Sprintf("%v.db", ps.dbprefix)) + if err != nil { + log.Fatal(err) + } + ps.DB = db + return nil +} +func (ps *PQLToSQL) MustRemove() { + name := fmt.Sprintf("%v.db", ps.dbprefix) + os.Remove(name) + // panicOn(err) +} + +func (ps *PQLToSQL) Stop() error { + return ps.DB.Close() +} +func (ps *PQLToSQL) Store(index, field string, rowIdOrKey interface{}, rowCall []*pql.Call) (bool, error) { + rowSql, _, _, err := ToSql(rowCall, index) + panicOn(err) + insertSql := fmt.Sprintf(`insert into %vλbits select "%v" as field, "%v" as row, column,0 as timestamp from ( %v )`, index, field, rowIdOrKey, rowSql) + res, err := ps.DB.Exec(insertSql) + if err != nil { + return false, err + } + aff, err := res.RowsAffected() + if err != nil { + return false, err + } + return aff > 0, nil +} + +// convert '-' to 'Θ' since db cannot have '-' in table names, +// but pilosa allows it in index and field names. +func dash2theta(s string) (r string) { + return strings.Replace(s, "-", "Θ", -1) +} + +func theta2dash(s string) (r string) { + return strings.Replace(s, "Θ", "-", -1) +} +func (ps *PQLToSQL) CreateSchema(jsonBytes []byte) (err error) { + err = json.Unmarshal(jsonBytes, &ps.schema) + if err != nil { + panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) + } + for _, i := range ps.schema.Indexes { + ps.createDatabase(i.Name) + for _, f := range i.Fields { + panicOn(ps.CreateField(i, f)) + } + } + return nil +} + +func (ps *PQLToSQL) CreateFieldWithOptions(indexName, fieldName string, o fieldOptions) error { + + ii2, ok := ps.i2f[indexName] + if !ok { + return fmt.Errorf("index '%v' not found", indexName) + } + + f2, ok := ii2.Fields[fieldName] + if !ok { + f2 = &FieldInfo2{ + Name: fieldName, + //CreatedAt: f.CreatedAt, + Options: *(o.ToPilosaFieldOptions()), + //Views: f.Views, + indexInfo: ii2, + } + ii2.Fields[fieldName] = f2 + } + return nil +} + +func (ps *PQLToSQL) CreateIndex(indexName string, opts *pilosa.IndexOptions) (err error) { + ii2, ok := ps.i2f[indexName] + if !ok { + ii2 = &IndexInfo2{ + Name: indexName, + //CreatedAt: i.CreatedAt, + Options: *opts, + Fields: make(map[string]*FieldInfo2), + ShardWidth: 1048576, + } + //TODO (twg) figure oiut where to get Shardwidht + ps.i2f[indexName] = ii2 + } + return nil +} +func (ps *PQLToSQL) Query(index string, query []*pql.Call) (err error) { + // sql, err := ToSQL(query, index) + //TODO (twg) implement QUery + return nil +} +func (ps *PQLToSQL) CreateField(i *pilosa.IndexInfo, f *pilosa.FieldInfo) (err error) { + ii2, ok := ps.i2f[i.Name] + if !ok { + ii2 = &IndexInfo2{ + Name: i.Name, + CreatedAt: i.CreatedAt, + Options: i.Options, + Fields: make(map[string]*FieldInfo2), + ShardWidth: i.ShardWidth, + } + ps.i2f[i.Name] = ii2 + } + + f2, ok := ii2.Fields[f.Name] + if !ok { + f2 = &FieldInfo2{ + Name: f.Name, + CreatedAt: f.CreatedAt, + Options: f.Options, + Views: f.Views, + indexInfo: ii2, + } + ii2.Fields[f.Name] = f2 + } + return nil +} + +func (ps *PQLToSQL) createDatabase(index string) (err error) { + idx_prefix := dash2theta(index) + "λ" + // Create fields that don't exist. + sql := fmt.Sprintf("create table %vbits (field string,row,column,timestamp, unique(field,row,column,timestamp) )", idx_prefix) + _, err = ps.DB.Exec(sql) + panicOn(err) + sql = fmt.Sprintf("create table %vbsi (field,column, val bigint,unique(field,column)) ", idx_prefix) + _, err = ps.DB.Exec(sql) + panicOn(err) + sql = fmt.Sprintf(`CREATE VIEW %vcolumns as + select distinct column from %vbits + UNION + select distinct column from %vbsi`, idx_prefix, idx_prefix, idx_prefix) + _, err = ps.DB.Exec(sql) + panicOn(err) + return +} +func (ps *PQLToSQL) GetField(indexName, fieldName string) (*FieldInfo2, error) { + idx, ok := ps.i2f[indexName] + if !ok { + return nil, errors.New(fmt.Sprintf("index not found '%v'", indexName)) + } + f, ok := idx.Fields[fieldName] + if !ok { + return nil, errors.New(fmt.Sprintf("field not found '%v' in index '%v'", fieldName, indexName)) + } + return f, nil +} + +//TODO (twg) below will be moved to its own file later just have it here for quick navigation +type PqlFunction uint8 + +//debate between this +type ResultType uint8 + +const ( + UNKNOWN = ResultType(iota) + BITMAP + INT + ARRAY + GROUPCOUNT + SIGNEDROW + BOOL + PAIRSFIELD + TABLE + AGGSQL +) + +func getAggregateSql(call *pql.Call, index string) (string, error) { + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", errors.New("no _field present") + } + } + var sq string + if len(call.Children) > 0 { + //only child should be a bitmap call + var err error + sq, _, _, err = ToSql(call.Children, index) + if err != nil { + return sq, err + } + sq = " and column in (" + sq + ")" + + } + sqb := fmt.Sprintf(`select %v(val) from %vλbsi where field="%v"%v`, strings.ToLower(call.Name), index, fieldName, sq) + sql := fmt.Sprintf(`select val,count(*) from %vλbsi where val=(%v)%v`, index, sqb, sq) + + return sql, nil +} + +// ToSql takes a pilosa query(ast) and converts it to sql +// assumption: the pql has successfully executed on server +func applyCondition(field string, cond *pql.Condition) (sql string) { + + switch cond.Op { + case pql.EQ, pql.NEQ, pql.LT, pql.LTE, pql.GT, pql.GTE: + val, ok := cond.Uint64Value() + if !ok { + panic("no value for condition") + } + if cond.Op == pql.EQ { + sql = fmt.Sprintf(`field="%v" and val=%d`, field, val) + } else if cond.Op == pql.NEQ { + sql = fmt.Sprintf(`field="%v" and val!=%d`, field, val) + } else if cond.Op == pql.LT { + sql = fmt.Sprintf(`field="%v" and val<%d`, field, val) + } else if cond.Op == pql.LTE { + sql = fmt.Sprintf(`field="%v" and val<=%d`, field, val) + } else if cond.Op == pql.GT { + sql = fmt.Sprintf(`field="%v" and val>%d`, field, val) + } else if cond.Op == pql.GTE { + sql = fmt.Sprintf(`field="%v" and val>=%d`, field, val) + } + case pql.BETWEEN, pql.BTWN_LT_LTE, pql.BTWN_LTE_LT, pql.BTWN_LT_LT: + val, ok := cond.Uint64SliceValue() + if !ok { + panic("bad value for condition") + } + if cond.Op == pql.BETWEEN { + sql = fmt.Sprintf(`field="%v" and %v>=%d and %v<=%d`, field, "val", val[0], "val", val[1]) + } else if cond.Op == pql.BTWN_LT_LTE { + sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[1]) + } else if cond.Op == pql.BTWN_LTE_LT { + sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[0]) + } else if cond.Op == pql.BTWN_LT_LT { + sql = fmt.Sprintf(`field="%v" and %v>%d and %v<%d`, field, "val", val[0], "val", val[1]) + } + } + return +} +func FetchFields(rowsCalls []*pql.Call) ([]string, []interface{}, error) { + //TODO (twg) date fields in rows? + results := make([]string, len(rowsCalls)) + columns := make([]interface{}, 0) + for i, call := range rowsCalls { + //TODO (twg) Time Fields ie from/to args + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return nil, nil, errors.New("no field for rows") + } + } + results[i] = fieldName.(string) + col, ok := call.Args["column"] + if ok { + s := fmt.Sprintf("f%v.column='%v'", i+1, col) + columns = append(columns, s) + } + } + return results, columns, nil +} + +func ToSql(ast []*pql.Call, index string) (string, ResultType, []interface{}, error) { + resultType := UNKNOWN + optional := make([]interface{}, 0) + if len(ast) == 0 { + return "", resultType, optional, nil + } + stack := make([]string, 0) + for _, call := range ast { + switch c := strings.ToLower(call.Name); c { + case "row": + var sql string + if call.HasConditionArg() { + for k, v := range call.Args { + csql := applyCondition(k, v.(*pql.Condition)) + sql = fmt.Sprintf("select distinct column from %vλbsi where %v", index, csql) + + break + } + } else { + field, err := call.FieldArg() + panicOn(err) + row := call.Args[field] + + t, ok := call.Args["from"] + var timestampClause string + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause = fmt.Sprintf(` AND timestamp >= "%v"`, clean) + } + t, ok = call.Args["to"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause += fmt.Sprintf(` AND timestamp < "%v"`, clean) + + } + + sql = fmt.Sprintf(`select distinct column from %vλbits where field="%v" AND row="%v"%v`, index, field, row, timestampClause) + + } + stack = append(stack, sql) + resultType = BITMAP + case "count": + //should have args and should be top level + if len(call.Children) == 0 { + return "", resultType, optional, errors.New("invalid args for count") + } + nestedSQL, _, _, err := ToSql(call.Children, index) + panicOn(err) + sql := "select count(*) from( " + nestedSQL + " )" + stack = append(stack, sql) + resultType = INT + case "union", "intersect": + var sql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(sql) == 0 { + sql = sq + } else { + sql += " " + c + " " + sq + } + } + sql = "select column from(" + sql + ")" + stack = append(stack, sql) + resultType = BITMAP + case "difference": + var sql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(sql) == 0 { + sql = sq + } else { + sql += " except " + sq + } + } + sql = "select column from(" + sql + ")" + stack = append(stack, sql) + resultType = BITMAP + case "not": + //should have args and should be top level + if len(call.Children) == 0 { + return "", resultType, optional, errors.New("invalid args for count") + } + nestedSQL, _, _, err := ToSql(call.Children, index) + panicOn(err) + sql := fmt.Sprintf("select column from %vλcolumns except %v", index, nestedSQL) + stack = append(stack, sql) + resultType = BITMAP + case "xor": + var unionSql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(unionSql) == 0 { + unionSql = sq + } else { + unionSql += " union " + sq + } + } + unionSql = "select column from(" + unionSql + ")" + var intersectSql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(intersectSql) == 0 { + intersectSql = sq + } else { + intersectSql += " intersect " + sq + } + } + intersectSql = "select column from(" + intersectSql + ")" + sql := unionSql + " except " + intersectSql + stack = append(stack, sql) + resultType = BITMAP + + case "distinct": + //TODO validate field type, only valid for bsi fields + fieldName, ok := call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("field not specified on distinct call") + } + var sql string + if len(call.Children) > 1 { + return "", resultType, optional, errors.New(fmt.Sprintf("unexpected args to distinct query:'%v'", call.Children)) + } + sql = fmt.Sprintf(`select distinct val from %vλbsi where field="%v"`, index, fieldName) + if len(call.Children) > 0 { + sq, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) + panicOn(err) + sql += " AND column in (" + sq + ")" + } + stack = append(stack, sql) + resultType = SIGNEDROW + case "rows": + //TODO (twg) Time Fields ie from/to args + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("no _field present") + } + } + optional = append(optional, fieldName) + sql := fmt.Sprintf(`select distinct field, row from %vλbits where field="%v"`, index, fieldName) + col, ok := call.Args["column"] + if ok { + sql = fmt.Sprintf("%v and column='%v'", sql, col) + } + prev, ok := call.Args["previous"] + if ok { + sql = fmt.Sprintf("%v and row>'%v'", sql, prev) + } + t, ok := call.Args["from"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) + sql += timestampClause + } + t, ok = call.Args["to"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) + sql += timestampClause + } + sql += " order by row" + limit, ok := call.Args["limit"] + if ok { + sql = fmt.Sprintf("%v limit %v", sql, limit) + } + stack = append(stack, sql) + resultType = ARRAY + case "groupby": + //need to figure out how many fields we will be looking at + fields, optColumns, err := FetchFields(call.Children) + panicOn(err) + sql := "select " + for i := range fields { + f := fmt.Sprintf("f%d", i+1) + if i == 0 { + sql += fmt.Sprintf(" %s.field, %s.row", f, f) + } else { + + sql += fmt.Sprintf(",%s.field, %s.row", f, f) + } + } + tname := fmt.Sprintf("%vλbits", index) + sql += fmt.Sprintf(",count(*) as cnt from %v f1", tname) + if len(fields) > 1 { + for i := range fields[1:] { + f := fmt.Sprintf("f%d", i+2) + sql += fmt.Sprintf(" inner join %v %v on f1.column = %v.column", tname, f, f) + } + } + sql += " where " + gb := "" + for i, field := range fields { + f := fmt.Sprintf("f%d", i+1) + if i == 0 { + sql += fmt.Sprintf("%v.field='%v'", f, field) + gb += fmt.Sprintf("%v.field,%v.row", f, f) + } else { + sql += fmt.Sprintf("and %v.field='%v'", f, field) + gb += fmt.Sprintf(",%v.field,%v.row", f, f) + } + } + if len(optColumns) > 0 { + for _, column := range optColumns { + sql += fmt.Sprintf(" and %v ", column) + } + } + sql += " group by " + gb + " order by cnt desc" + + optional = append(optional, len(fields)) + + stack = append(stack, sql) + resultType = GROUPCOUNT + case "topn": + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("no _field present") + } + } + var sq string + if len(call.Children) > 0 { + if len(call.Children) > 1 { + return "", resultType, optional, errors.New("invalid topn") + } + var err error + sq, _, _, err = ToSql(call.Children, index) + panicOn(err) + sq = " and column in (" + sq + ")" + + } + sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) + + limit, ok := call.Args["n"] + if ok { + sql += fmt.Sprintf(" limit %v", limit) + } + stack = append(stack, sql) + resultType = PAIRSFIELD + case "topk": + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("no _field present") + } + } + var sq string + filter, ok := call.Args["filter"] + if ok { + //assuming this works in pilosa so the filter must be a correct bitmap call + //ie row/union/intersect/xor/difference + var err error + sq, _, _, err = ToSql([]*pql.Call{filter.(*pql.Call)}, index) + panicOn(err) + sq = " and column in (" + sq + ")" + + } + t, ok := call.Args["from"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) + sq += timestampClause + } + t, ok = call.Args["to"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) + sq += timestampClause + } + sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) + + limit, ok := call.Args["k"] + if ok { + sql += fmt.Sprintf(" limit %v", limit) + } + stack = append(stack, sql) + resultType = PAIRSFIELD + case "min", "max": + sql, err := getAggregateSql(call, index) + panicOn(err) + stack = append(stack, sql) + resultType = AGGSQL + case "sum": + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + panic("no _field present") + } + } + var sq string + if len(call.Children) > 0 { + //only child should be a bitmap call + var err error + sq, _, _, err = ToSql(call.Children, index) + panicOn(err) + sq = " and column in (" + sq + ")" + + } + sql := fmt.Sprintf(`select sum(val),count(*) from %vλbsi where field="%v"%v`, index, fieldName, sq) + stack = append(stack, sql) + resultType = AGGSQL + case "constrow": + sql := "select column1 as column from (values " + columns, ok := call.Args["columns"].([]interface{}) + + if ok { + for i, col := range columns { + if i != 0 { + sql += "," + } + sql += fmt.Sprintf("(%v)", col) + + } + sql += ")" + + } + + stack = append(stack, sql) + resultType = BITMAP + case "all": + sql := fmt.Sprintf("select column from %vλcolumns", index) + stack = append(stack, sql) + resultType = BITMAP + case "extract": + // extract is a table response + bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) + panicOn(err) + var rowsPart string + if len(call.Children) > 1 { + rows := call.Children[1:] + rowsPart = "select field,row from(" + for i, row := range rows { + part, _, _, err1 := ToSql([]*pql.Call{row}, index) + panicOn(err1) + if i > 0 { + rowsPart += "union all select field,row from (" + } + rowsPart += part + ")" + } + } + var sql string + if len(rowsPart) == 0 { + sql = "select '' as field, '' as row,column from (" + bm + ")" + } else { + sql = "select sq.field, sq.row, b.column from (" + sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) + sql += " where b.column in (" + bm + ")" + } + + stack = append(stack, sql) + resultType = TABLE + case "unionrows": + var rowsPart string + rowsPart = "select field,row from(" + for i, row := range call.Children { + part, _, _, err1 := ToSql([]*pql.Call{row}, index) + panicOn(err1) + if i > 0 { + rowsPart += "union all select field,row from (" + } + rowsPart += part + ")" + } + sql := "select distinct b.column from (" + sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) + stack = append(stack, sql) + resultType = BITMAP + case "includescolumn": + bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) + panicOn(err) + col, ok := call.Args["column"] + if !ok { + return "", resultType, optional, errors.New("missing column parameter") + } + + sql := fmt.Sprintf("select count(*)>0 from (select column from(%v) where column='%v')", bm, col) + stack = append(stack, sql) + resultType = BOOL + default: + goon.Dump(call) + return "", resultType, optional, errors.New(fmt.Sprintf("unknown call '%v'", c)) + } + } + if len(stack) == 0 { + //not sure if i should error on an NOP + return "", resultType, optional, nil + } + return stack[0], resultType, optional, nil +} + +func NewSauron(cfg *SauronConfig) *Sauron { + + addr := "127.0.0.1:8080" + url := "http://" + addr + s := &Sauron{ + cfg: cfg, + addr: addr, + url: url, + sql: NewPQLToSQL(cfg.DatabaseName), + srv: &http.Server{ + Addr: addr, + ReadTimeout: 50 * time.Millisecond, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + }, + } + s.Handler = newRouter(s) + s.srv.Handler = s.Handler + return s +} + +func (p *Sauron) Start() (urlstring string, err error) { + started := make(chan struct{}) + + panicOn(p.sql.Start()) + + // start a pilosa server + bind := p.cfg.Bind + dataDir := p.cfg.DataDir + bindGRPC := p.cfg.BindGRPC + gossipPort := p.cfg.GossipPort + + p.cmd = exec.Command("pilosa", "server", + "--bind", bind, + "--data-dir", dataDir, + "--bind-grpc", bindGRPC, + "--gossip.port", gossipPort) + err = p.cmd.Start() + panicOn(err) + + // wait for it to be listening + p.pilosaHttpURL = "http://" + bind + ok := false + seconds := 20 + for i := 0; i < seconds; i++ { + nc, err := net.Dial("tcp", bind) + if err == nil { + nc.Close() + ok = true + break + } + time.Sleep(time.Second) + } + if !ok { + return "", fmt.Errorf("could not contact pilosa server on %v after %v seconds. pid = %v", bind, seconds, p.cmd.Process.Pid) + } + go func() { + close(started) + + err = p.srv.ListenAndServe() + if err != nil { + if strings.Contains(err.Error(), "Server closed") { + return + } + } + panicOn(err) + }() + <-started + return p.url, nil +} + +func (p *Sauron) Stop() { + panicOn(p.sql.Stop()) + err := p.srv.Shutdown(context.Background()) + panicOn(err) + // Kill pilosa + if err = p.cmd.Process.Kill(); err != nil { + vv("failed to kill process: ", err) + } +} + +// GetAvailPort asks the OS for an unused port. +// There's a race here, where the port could be grabbed by someone else +// before the caller gets to Listen on it, but we are only using +// it to find a random port for the test hang debugging. +// Moreover, in practice such races are rare. Just ask for +// it again if the port is taken. +// Uses net.Listen("tcp", ":0") to determine a free port, then +// releases it back to the OS with Listener.Close(). +func GetAvailPort() int { + l, _ := net.Listen("tcp", ":0") + r := l.Addr() + l.Close() + return r.(*net.TCPAddr).Port +} + +// newRouter creates a new mux http router. +func newRouter(handler *Sauron) http.Handler { + router := mux.NewRouter() + + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") + router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") + router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") + + router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex") + router.HandleFunc("/index/{index}/field/", handler.handlePostField).Methods("POST").Name("PostField") + router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField") + router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring") + router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport") + + return router +} + +// handleGetSchema handles GET /schema requests. +func (s *Sauron) handleGetSchema(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + _ = body + _ = resp + _ = err + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(resp.StatusCode) + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + _, err = w.Write(respBody) + panicOn(err) + + // B) there is no SQL version of get schema. + +} + +// creates an entire schema from the dumped schema. +// requires completed structures, no defaults applied +func (s *Sauron) handlePostSchema(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + + w.WriteHeader(resp.StatusCode) + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + _, err = w.Write(respBody) + panicOn(err) + + // B) SQL + panicOn(s.sql.CreateSchema(body)) +} + +// handlePostField handles /index/{index}/field/{field} requests +func (s *Sauron) handlePostIndex(w http.ResponseWriter, r *http.Request) { + body, resp, err := s.ForwardHandler(w, r) + _ = body + _ = resp + _ = err + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + indexName, ok := mux.Vars(r)["index"] + if !ok { + http.Error(w, "index name is required", http.StatusBadRequest) + return + } + req := struct { + Options pilosa.IndexOptions + }{ + Options: pilosa.IndexOptions{ + Keys: false, + TrackExistence: true, + }} + if len(body) > 0 { //only process if options are present + err = json.Unmarshal(body, &req) + panicOn(err) + } + err = s.sql.CreateIndex(indexName, &req.Options) + panicOn(err) + w.Write(respBody) +} + +// handlePostField handles /index/{index}/field/{field} requests +func (s *Sauron) handlePostField(w http.ResponseWriter, r *http.Request) { + body, resp, err := s.ForwardHandler(w, r) + _ = body + _ = resp + _ = err + indexName, ok := mux.Vars(r)["index"] + if !ok { + http.Error(w, "index name is required", http.StatusBadRequest) + return + } + + fieldName, ok := mux.Vars(r)["field"] + if !ok { + http.Error(w, "field name is required", http.StatusBadRequest) + return + } + + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + // Decode request. + var req postFieldRequest + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + err = dec.Decode(&req) + if err != io.EOF { + panicOn(err) + } + err = s.sql.CreateFieldWithOptions(indexName, fieldName, req.Options) + _, err = w.Write(respBody) + panicOn(err) +} + +// handlePostQuery handles /index/{index}/query requests +func (s *Sauron) handlePostQuery(w http.ResponseWriter, req *http.Request) { + + body, resp, err := s.ForwardHandler(w, req) + pilosaJSON, err := ioutil.ReadAll(resp.Body) + panicOn(err) + var pilosaResponse pilosa.QueryResponse // DEBUG + dec := json.NewDecoder(bytes.NewReader(pilosaJSON)) + err = dec.Decode(&pilosaResponse) + panicOn(err) + qreq := &pilosa.QueryRequest{ + Index: mux.Vars(req)["index"], + Query: string(body), + } + + oracleResponse, err := s.sql.IssueQuery(qreq) + + if !pilosaResponse.Equals(oracleResponse) { + //plan is to write out a useful query response indicating a discrepancy + //TODO (twg) write diff in response somehow + vv("oracle:") + prettyJson(*oracleResponse) + vv("pilosa:") + prettyJson(pilosaResponse) + panic("barf") + } +} +func prettyJson(a pilosa.QueryResponse) { + src, _ := a.MarshalJSON() + dst := &bytes.Buffer{} + if err := json.Indent(dst, src, "", " "); err != nil { + panic(err) + } + vv("%v", string(dst.Bytes())) +} + +func (s *Sauron) FieldInfo(index, field string) (*FieldInfo2, error) { + idx, ok := s.sql.i2f[index] + if ok { + f, ok := idx.Fields[field] + if ok { + return f, nil + } + return nil, errors.New(fmt.Sprintf("field not in schema '%v'", field)) + } + return nil, errors.New(fmt.Sprintf("index not in schema '%v'", index)) +} +func (s *Sauron) handlePostImport(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + _ = body + _ = resp + _ = err + // Get index and field type to determine how to handle the + // import data. + indexName := mux.Vars(req)["index"] + fieldName := mux.Vars(req)["field"] + + _ = indexName + _ = fieldName + + // If the clear flag is true, treat the import as clear bits. + q := req.URL.Query() + doClear := q.Get("clear") == "true" + doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" + _ = doClear + _ = doIgnoreKeyCheck + + info, err := s.FieldInfo(indexName, fieldName) + panicOn(err) + // Unmarshal request based on field type. + fieldType := info.Options.Type + if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal { + iv := &pilosa.ImportValueRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, iv); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + //TODO (twg) ignore columnKeys at the moment + //do importValue + for i, column := range iv.ColumnIDs { + if len(iv.Values) > 0 { + s.sql.Set(indexName, column, fieldName, iv.Values[i], "") + } else if len(iv.FloatValues) > 0 { //TODO (twg) these may not be reachable + s.sql.Set(indexName, column, fieldName, iv.FloatValues[i], "") //probably some issues here + } else if len(iv.StringValues) > 0 { + s.sql.Set(indexName, column, fieldName, iv.StringValues[i], "") + } + } + } else { + // Field type: set, time, mutex + // Marshal into request object. + ir := &pilosa.ImportRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, ir); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + //do import + action := s.sql.Set + if ir.Clear { + action = func(index string, column uint64, field string, value interface{}, ts string) (bool, error) { + return s.sql.Clear(index, column, field, value) + } + } + changed := false + _ = changed + for i, column := range ir.ColumnIDs { + ts := "" + if len(ir.Timestamps) > 0 { + t := time.Unix(ir.Timestamps[i], 0) + ts = t.Format("2006-01-02 15:04:05") + } + if len(ir.RowIDs) > 0 { + change, err := action(indexName, column, fieldName, ir.RowIDs[i], ts) + panicOn(err) + if change { + changed = true + } + } else if len(ir.RowKeys) > 0 { //TODO (twg) these may not be reachable + change, err := action(indexName, column, fieldName, ir.RowKeys[i], ts) + panicOn(err) + if change { + changed = true + } + } + } + } +} +func (s *Sauron) handlePostImportRoaring(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + _ = body + _ = resp + _ = err + // Get index and field type to determine how to handle the + // import data. + indexName := mux.Vars(req)["index"] + fieldName := mux.Vars(req)["field"] + + irr := &pilosa.ImportRoaringRequest{} + err = proto.DefaultSerializer.Unmarshal(body, irr) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + //do i need the shard? + urlVars := mux.Vars(req) + shard, err := strconv.ParseUint(urlVars["shard"], 10, 64) + if err != nil { + panicOn(err) + } + for v, blob := range irr.Views { + _ = v //ignore the view for now + rit, err := roaring.NewRoaringIterator(blob) + panicOn(err) + s.sql.ImportRoaring(indexName, fieldName, shard, rit) + } +} + +//end handler +func (s *Sauron) ForwardHandler(w http.ResponseWriter, req *http.Request) (body []byte, resp *http.Response, err error) { + // TODO (twg) replace all the instances + // we need to buffer the body if we want to read it here and send it + // in the request. + body, err = ioutil.ReadAll(req.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // you can reassign the body if you need to parse it as multipart + req.Body = ioutil.NopCloser(bytes.NewReader(body)) + + // create a new url from the raw RequestURI sent by the client + url := fmt.Sprintf("%s%s", s.pilosaHttpURL, req.RequestURI) + + proxyReq, err := http.NewRequest(req.Method, url, bytes.NewReader(body)) + + // We may want to filter some headers, otherwise we could just use a shallow copy + // proxyReq.Header = req.Header + proxyReq.Header = make(http.Header) + for h, val := range req.Header { + proxyReq.Header[h] = val + } + + resp, err = http.DefaultClient.Do(proxyReq) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + return +} + +// ImportRoaring must not be a bsi field..check higher up the stack +func (ps *PQLToSQL) ImportRoaring(index, field string, shard uint64, itr roaring.RoaringIterator) { + rb := roaring.NewBitmap() + for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { + rb.Put(itrKey, synthC) + } + for _, pos := range rb.Slice() { + row, column := pos/ps.ShardWidth, (shard*ps.ShardWidth)+(pos%ps.ShardWidth) + //ps.SetWithInt64Field(index, column, field, int64(row)) + ps.Set(index, column, field, int64(row), "") + } +} + +func (ps *PQLToSQL) isBSI(index, field string) (full string, isBSI bool) { + idx, ok := ps.i2f[index] + if !ok { + panic(fmt.Errorf("isBSI: no such index '%v'", index)) + } + + fld, ok := idx.Fields[field] + if !ok { + panic(fmt.Errorf("isBSI: no such field '%v'", field)) + } + if fld.Options.Type == "int" { + return dash2theta(index) + "λbsi", true + } + return dash2theta(index) + "λbits", false +} + +// Set is the basic bit operation and value operation as well, depending on the field type +func (ps *PQLToSQL) Set(index string, column uint64, field string, value interface{}, timestamp string) (bool, error) { + table, isBSI := ps.isBSI(index, field) + var sql string + if isBSI { + sql = fmt.Sprintf("INSERT INTO %v VALUES( '%v','%v', '%v')", table, field, column, value) + } else { + //if len(timestamp) == 0 { + // sql = fmt.Sprintf("INSERT INTO %v (field,row,column)VALUES( '%v','%v', '%v')", table, field, value, column) + // } else { + //TODO(twg) will need to make the timestamp as granular as the field is configured. + sql = fmt.Sprintf("INSERT INTO %v (field,row,column,timestamp)VALUES( '%v','%v', '%v','%v')", table, field, value, column, timestamp) + // } + } + + _, err := ps.DB.Exec(sql) + if err != nil { + if sqerr, ok := err.(*sqlite.Error); ok { + if sqerr.Code() == 2067 { //constraint error + return false, nil + } + } + } + panicOn(err) + return true, nil +} + +// Clear removes the specified column or value +func (ps *PQLToSQL) Clear(index string, column uint64, field string, value interface{}) (bool, error) { + table, isBSI := ps.isBSI(index, field) + var sql string + if isBSI { + sql = fmt.Sprintf("delete from %v where field='%v' and column='%v'", table, field, column) + } else { + sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' and column='%v'", table, field, value, column) + } + + res, err := ps.DB.Exec(sql) + panicOn(err) + r, err := res.RowsAffected() + panicOn(err) + if r == 0 { + return false, nil + } + return true, nil +} + +// ClearRow removes entire Row(feature) +func (ps *PQLToSQL) ClearRow(index string, field string, row interface{}) (bool, error) { + table, isBSI := ps.isBSI(index, field) + var sql string + if isBSI { + panic("bsi can't clear row allowed") + } else { + sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' ", table, field, row) + } + + r, err := ps.DB.Exec(sql) + panicOn(err) + n, err := r.RowsAffected() + return n > 0, err +} + +func sliceToString(slc []int64) (r string) { + if len(slc) == 0 { + return + } + for _, i := range slc { + r += fmt.Sprintf("%v,", i) + } + return r[:len(r)-1] +} + +const columnLabel = "col" + +func (ps *PQLToSQL) IssueQuery(req *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { + index := req.Index + q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() + if err != nil { + return nil, errors.Wrap(err, "PQLToSQL.IssueQuery parsing PQL") + } + var results pilosa.QueryResponse + for _, c := range q.Calls { + + if callIndex := c.CallIndex(); callIndex != "" { + index = callIndex + } + + // Handle the field arg. + switch strings.ToLower(c.Name) { + //handle the write queries + //TODO (twg) figure out sqlite changes() function to determine number of bits changed + case "set": + // Read colID. + colID, ok, err := c.UintArg("_" + columnLabel) + if err != nil { + return nil, fmt.Errorf("reading Set() column: %v", err) + } else if !ok { + return nil, fmt.Errorf("Set() column argument '%v' required", columnLabel) + } + ts, ok := c.Args["_timestamp"] + var timestamp string + if ok { + timestamp = strings.Replace(ts.(string), "T", " ", 1) + } + + field, err := c.FieldArg() + if err == nil { + rs, err := ps.Set(index, colID, field, c.Args[field], timestamp) + panicOn(err) + results.Results = append(results.Results, rs) + } else { + return nil, err + } + case "clear": + // Read colID. + colID, ok, err := c.UintArg("_" + columnLabel) + if err != nil { + return nil, fmt.Errorf("reading Clear() column: %v", err) + } else if !ok { + return nil, fmt.Errorf("Clear() column argument '%v' required", columnLabel) + } + + field, err := c.FieldArg() + if err == nil { + rs, err := ps.Clear(index, colID, field, c.Args[field]) + if err != nil { + return nil, fmt.Errorf("Clear() column: %v", err) + } + results.Results = append(results.Results, rs) + } else { + vv("c.FieldArg() returned '%v'", err) + } + case "clearrow": + field, err := c.FieldArg() + panicOn(err) + rs, err := ps.ClearRow(index, field, c.Args[field]) + panicOn(err) + results.Results = append(results.Results, rs) + case "store": + field, err := c.FieldArg() + panicOn(err) + rs, err := ps.Store(index, field, c.Args[field], c.Children) + panicOn(err) + results.Results = append(results.Results, rs) + default: //TODO (twg) move ToSQL into main package + rs, err := ps.readQuery(index, q) + panicOn(err) + results.Results = append(results.Results, rs) + } + } + return &results, nil +} +func (ps *PQLToSQL) handleBitmapCall(sql string) (*pilosa.Row, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + defer rows.Close() + columns := make([]uint64, 0) + for rows.Next() { + //really need a way to know what the results look like + var column uint64 + rows.Scan(&column) + columns = append(columns, column) + } + row := pilosa.NewRow(columns...) + return row, nil +} + +func (ps *PQLToSQL) handleIntCall(sql string) (count uint64, err error) { + rows, err := ps.DB.Query(sql) + if err != nil { + return 0, err + } + defer rows.Close() + for rows.Next() { + //really need a way to know what the results look like + rows.Scan(&count) + return + } + return +} + +type Table struct { +} + +func (ps *PQLToSQL) handleSignedRowCall(sql string) (pilosa.SignedRow, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + + var id uint64 + var pos []uint64 + var neg []uint64 + defer rows.Close() + for rows.Next() { + err := rows.Scan(&id) + if err != nil { + return pilosa.SignedRow{}, err + } + if id >= 0 { + pos = append(pos, id) + } else { + neg = append(neg, id) + } + } + err = rows.Err() + if err != nil { + return pilosa.SignedRow{}, err + } + posRow := pilosa.NewRow(pos...) + negRow := pilosa.NewRow(neg...) + return pilosa.SignedRow{Pos: posRow, Neg: negRow}, nil + +} +func (ps *PQLToSQL) handleStringArrayCall(sql string) ([]interface{}, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + rowKeys := make([]interface{}, 0) + var field string + var id string + defer rows.Close() + for rows.Next() { + err := rows.Scan(&field, &id) + if err != nil { + return nil, err + } + rowKeys = append(rowKeys, id) + } + err = rows.Err() + if err != nil { + return nil, err + } + return rowKeys, nil +} + +func (ps *PQLToSQL) handleIntArrayCall(sql string) (map[string][]interface{}, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + rowIDs := make([]interface{}, 0) + var id uint64 + var field string + defer rows.Close() + for rows.Next() { + err := rows.Scan(&field, &id) + if err != nil { + return nil, err + } + rowIDs = append(rowIDs, id) + } + err = rows.Err() + if err != nil { + return nil, err + } + m := map[string][]interface{}{"rows": rowIDs, "keys": []interface{}{}} + return m, nil +} +func (ps *PQLToSQL) handleGroupCountCall(sqlin string, index string, numFields int) (pilosa.SortedGroupCount, error) { + rows, err := ps.DB.Query(sqlin) + panicOn(err) + //groupCounts := make([]pilosa.GroupCount, 0) + var groupCounts pilosa.SortedGroupCount + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + return nil, err + } + row := make([]interface{}, len(cols)) + for i := range cols { + row[i] = new(sql.RawBytes) + } + for rows.Next() { + // need a row*NumFields + cnt + //err := rows.Scan(&field, &row, &count) + groupCount := pilosa.GroupCount{} + + err := rows.Scan(row...) + if err != nil { + return nil, err + } + for i := 0; i < len(row)-1; i += 2 { + ptr := row[i].(*sql.RawBytes) + fieldName := string(*ptr) + fr := pilosa.FieldRow{Field: fieldName} + fi, err := ps.GetField(index, fieldName) + if err != nil { + return nil, err + } + ptr = row[i+1].(*sql.RawBytes) + v := string(*ptr) + if fi.Options.Keys { + fr.RowKey = v + + } else { + iv, err := strconv.Atoi(v) + if err != nil { + return nil, err + } + fr.RowID = uint64(iv) + } + + groupCount.Group = append(groupCount.Group, fr) + + } + ptr := row[len(row)-1].(*sql.RawBytes) + data := string(*ptr) //convert to string + i, err := strconv.ParseInt(data, 10, 64) + panicOn(err) + groupCount.Count = uint64(i) + groupCounts = append(groupCounts, groupCount) + + } + + err = rows.Err() + if err != nil { + return nil, err + } + return groupCounts, nil +} +func (ps *PQLToSQL) handlePairsFieldCall(sql, index string) (pilosa.PairsField, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + var count uint64 + var row string + defer rows.Close() + res := pilosa.PairsField{} + for rows.Next() { + err := rows.Scan(&row, &count) + if err != nil { + return pilosa.PairsField{}, err + } + pair := pilosa.Pair{Key: row, Count: count} + res.Pairs = append(res.Pairs, pair) + } + err = rows.Err() + if err != nil { + return pilosa.PairsField{}, err + } + return res, nil +} + +// special structures to deal with Extract. In the future I would like to use +// pilosa structs, but dealing with keys makes that complicated for now +// note the column is a string, but i only worry about the type when marshalling out for +// the sqlite part is typeless. Handling keys is almost seemless and not requiring translation +type extractColumns struct { + column string + rows map[int][]interface{} + parent *sauronExtractedIDMatrix +} + +func (eid extractColumns) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + buf.WriteString(fmt.Sprintf(`{"column": %v,"rows":[`, eid.column)) + rows := make([][]interface{}, len(eid.parent.fields)) + for k, v := range eid.rows { + rows[k] = v + } + for i, grow := range rows { + if i != 0 { + buf.WriteByte(44) //write the comma + } + isKey := eid.parent.fields[i].isKey + buf.WriteString(`[`) + for d, row := range grow { + if d != 0 { + buf.WriteByte(44) //write the comma + } + if isKey { + buf.WriteString(fmt.Sprintf(`"%v"`, row)) + } else { + buf.WriteString(fmt.Sprintf(`%v`, row)) + } + + } + buf.WriteString(`]`) + } + + buf.WriteString(`]}`) + return buf.Bytes(), nil +} + +type extractFields struct { + Name string `json:"name"` + Type string `json:"type"` + isKey bool +} +type sauronExtractedIDMatrix struct { + columns []extractColumns + fields []extractFields +} + +func (eid sauronExtractedIDMatrix) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + sort.Slice(eid.columns, func(i, j int) bool { + return eid.columns[i].column < eid.columns[j].column + }) + buf.WriteString(`{"columns":[`) + for i, col := range eid.columns { + b, err := col.MarshalJSON() + panicOn(err) + if i != 0 { + buf.WriteByte(44) //write the comma + } + buf.Write(b) + } + + buf.WriteString(`],"fields":[`) + for i, f := range eid.fields { + if i != 0 { + buf.WriteByte(44) + } + b, err := json.Marshal(f) + panicOn(err) + buf.Write(b) + } + buf.WriteString(`]}`) + return buf.Bytes(), nil +} + +func (ps *PQLToSQL) handleTableCall(sql, index string) (sauronExtractedIDMatrix, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + var field string + var row string + var column string + defer rows.Close() + col := make(map[string]map[string][]interface{}) + fidx := make(map[string]int) + results := sauronExtractedIDMatrix{} + counter := 0 + empty := make(map[string][]interface{}) + for rows.Next() { + err := rows.Scan(&field, &row, &column) + if err != nil { + return sauronExtractedIDMatrix{}, err + } + if field != "" { + _, ok := fidx[field] + if !ok { + fidx[field] = counter + counter++ + } + rows, ok := col[column] + if !ok { + rows = make(map[string][]interface{}) + } + rows[field] = append(rows[field], row) + //results.Columns = append(results.Columns, pilosa.ExtractedIDColumn{ColumnID: column, Rows: [][]uint64{}}) + col[column] = rows + } else { //constrow, or no filed info + col[column] = empty + } + } + err = rows.Err() + if err != nil { + return sauronExtractedIDMatrix{}, err + } + for c, rowMap := range col { + idc := extractColumns{rows: make(map[int][]interface{}), parent: &results} + idc.column = c + for fld, rows := range rowMap { + idx := fidx[fld] + idc.rows[idx] = rows + } + results.columns = append(results.columns, idc) + } + results.fields = make([]extractFields, len(fidx)) + for fieldName, idx := range fidx { + fi, _ := ps.GetField(index, fieldName) + // TODO (twg) hack need to figure out proper way to determine type + aType := "[]uint64" + if fi.Options.Keys { + aType = "[]string" + } + results.fields[idx] = extractFields{Name: fieldName, Type: aType, isKey: fi.Options.Keys} + } + + return results, nil +} + +func (ps *PQLToSQL) handleAggSQLCall(sql, index string) (pilosa.ValCount, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + defer rows.Close() + results := pilosa.ValCount{} + var val int64 + var count int64 + for rows.Next() { + err := rows.Scan(&val, &count) + if err != nil { + return pilosa.ValCount{}, err + } + results.Count = count + results.Val = val + break //shouldn't be anymore rows but being safe + } + err = rows.Err() + if err != nil { + return pilosa.ValCount{}, err + } + return results, nil +} +func (ps *PQLToSQL) handleBoolCall(sql, index string) (bool, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + defer rows.Close() + var val bool + for rows.Next() { + err := rows.Scan(&val) + if err != nil { + return false, err + } + break //shouldn't be anymore rows but being safe + } + err = rows.Err() + if err != nil { + return val, err + } + return val, nil +} + +func (ps *PQLToSQL) readQuery(index string, query *pql.Query) (interface{}, error) { + //Row(color=red) + sql, resultType, optional, err := ToSql(query.Calls, index) + panicOn(err) + switch resultType { + case BITMAP: + return ps.handleBitmapCall(sql) + case INT: + return ps.handleIntCall(sql) + case SIGNEDROW: + return ps.handleSignedRowCall(sql) + case ARRAY: + fieldName := optional[0].(string) + info, ok := ps.i2f[index] + if !ok { + return nil, errors.New(fmt.Sprintf("index not in schema %v", index)) + } + field, ok := info.Fields[fieldName] + if !ok { + return nil, errors.New(fmt.Sprintf("field not in schema %v", fieldName)) + } + if field.Options.Keys { + return ps.handleStringArrayCall(sql) + } else { + return ps.handleIntArrayCall(sql) + } + case GROUPCOUNT: + return ps.handleGroupCountCall(sql, index, optional[0].(int)) + case PAIRSFIELD: + return ps.handlePairsFieldCall(sql, index) + case TABLE: + return ps.handleTableCall(sql, index) + case AGGSQL: + return ps.handleAggSQLCall(sql, index) + case BOOL: + return ps.handleBoolCall(sql, index) + default: + vv("Unknown query Type %v", resultType) + } + return nil, nil +} + +type Int64Slice []int64 + +func (p Int64Slice) Len() int { return len(p) } +func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p Int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +// on tables starting with ps.dbprefix will be returned. +func (ps *PQLToSQL) ListIndexFields() (tables []string) { + for indexName, index := range ps.i2f { + for fieldname, _ := range index.Fields { + + tables = append(tables, indexName+"/"+fieldname) + } + } + sort.Strings(tables) + return tables +} diff --git a/cmd/sauron/populate.sh b/cmd/sauron/populate.sh new file mode 100644 index 000000000..46e7dae67 --- /dev/null +++ b/cmd/sauron/populate.sh @@ -0,0 +1,10 @@ +#!/bin/bash +mod=20 +for i in `seq 100 1`;do +b=$(dc -e "$i $mod %p") +[[ $b == 0 ]] && mod=$(($mod+1)) +r=$(($b+30)) +payload="Set($i, j=$r)" +echo $payload +curl -XPOST "localhost:10101/index/i/query" -d "$payload" +done diff --git a/cmd/sauron/sample_schema.json b/cmd/sauron/sample_schema.json new file mode 100644 index 000000000..4dcc2c501 --- /dev/null +++ b/cmd/sauron/sample_schema.json @@ -0,0 +1 @@ +{"indexes":[{"name":"trait_store","createdAt":1595896639330112903,"options":{"keys":true,"trackExistence":true},"fields":[{"name":"aba","createdAt":1595896641512346986,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"ach_batch_recency","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_pass_thru_recency","createdAt":1595896640333428009,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_payment_recency","createdAt":1595896642230184052,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"bools","createdAt":1595896640151751600,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"bools-exists","createdAt":1595896640855732760,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"central_group","createdAt":1595896640036032355,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"custom_audiences","createdAt":1595896643284443107,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"days_since_last_logon","createdAt":1595896639332677618,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":1,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"db","createdAt":1595896639548206413,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"desktop_recency","createdAt":1595896642633713852,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"domestic_wire_recency","createdAt":1595896642264402807,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"external_transfer_recency","createdAt":1595896639332854139,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"fields","createdAt":1595896639552562813,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"funds_transfer_recency","createdAt":1595896640803192694,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"international_wire_recency","createdAt":1595896643511015744,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"mobile_remote_deposit_recency","createdAt":1595896641174868205,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"payroll_recency","createdAt":1595896642880416952,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"pfm_category_total_current_balance__401k_investment","createdAt":1595896641492935339,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__403b_investment","createdAt":1595896639553865549,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__529_investment","createdAt":1595896643478533291,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__any","createdAt":1595896642202343868,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__auto_loan","createdAt":1595896642260798767,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__brokerage_account","createdAt":1595896639550130078,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__certificate_of_deposit","createdAt":1595896643210723905,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__checking","createdAt":1595896640763548235,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":42,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__credit_card","createdAt":1595896639734800167,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__home_equity_loan","createdAt":1595896642644368353,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__ira_investment","createdAt":1595896642076795921,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__line_of_credit","createdAt":1595896642071432710,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__loan","createdAt":1595896643101755055,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__money_market","createdAt":1595896642574933677,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":36,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__mortgage","createdAt":1595896640934767155,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__personal_loan","createdAt":1595896641378825138,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__roth_ira_investment","createdAt":1595896642821349899,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__savings","createdAt":1595896639534597334,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__simple_ira","createdAt":1595896639952200646,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__student_loan","createdAt":1595896642329394180,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__taxable_investment","createdAt":1595896641406468398,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"phone_recency","createdAt":1595896639708002303,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_count__brokerage_account","createdAt":1595896642501159753,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_cd_or_share_certificate","createdAt":1595896643269910233,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_checking_or_share_draft","createdAt":1595896640044097611,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_credit_card","createdAt":1595896641500609762,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_indirect_auto_loan","createdAt":1595896643353298900,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_line_of_credit","createdAt":1595896641219946523,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_loan","createdAt":1595896641263557904,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_money_market","createdAt":1595896642506172994,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_new_auto_loan","createdAt":1595896640791087766,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_real_estate_loan","createdAt":1595896642616627035,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_savings_or_share","createdAt":1595896642313967229,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_credit_card","createdAt":1595896642254165546,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_loan","createdAt":1595896642870853670,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_checking_or_share_draft","createdAt":1595896639333642847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_credit_card","createdAt":1595896642761030299,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_savings_or_share","createdAt":1595896643227769780,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_vehicle_loan","createdAt":1595896642991171391,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_used_auto_loan","createdAt":1595896642865599530,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_basic_checking_or_share_draft","createdAt":1595896640110059946,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_cd_or_share_certificate","createdAt":1595896640174468431,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_credit_card","createdAt":1595896640192571502,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_indirect_auto_loan","createdAt":1595896640563441892,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896643543919820,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_line_of_credit","createdAt":1595896640188449923,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_loan","createdAt":1595896639761090179,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_money_market","createdAt":1595896642731261748,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_new_auto_loan","createdAt":1595896643143947051,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_checking_or_share_draft","createdAt":1595896640346982139,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_savings_or_share","createdAt":1595896641346726397,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_real_estate_loan","createdAt":1595896643351578694,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_checking_or_share_draft","createdAt":1595896640302899735,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_savings_or_share","createdAt":1595896642726551847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_savings_or_share","createdAt":1595896642961872488,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_credit_card","createdAt":1595896640022927417,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_loan","createdAt":1595896642352438551,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_checking_or_share_draft","createdAt":1595896640804808345,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_credit_card","createdAt":1595896641113696262,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_savings_or_share","createdAt":1595896640312498993,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_vehicle_loan","createdAt":1595896639886308884,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_used_auto_loan","createdAt":1595896642412302198,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__debit_card","createdAt":1595896643127876947,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__education_savings_or_share","createdAt":1595896639338093281,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__escrow_account","createdAt":1595896642969874721,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__health_savings_account","createdAt":1595896640570458731,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_line_of_credit","createdAt":1595896642592520668,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_loan","createdAt":1595896643091627581,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__insurance_product","createdAt":1595896639744476133,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__interest_only_legal_trust_account","createdAt":1595896643005843869,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__membership_share","createdAt":1595896641732585653,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__other_service","createdAt":1595896641807551269,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_401k","createdAt":1595896639333271302,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":11,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_403b","createdAt":1595896643165638942,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_529","createdAt":1595896641640608805,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira","createdAt":1595896639555618615,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira_cd_or_share_certificate","createdAt":1595896641504460031,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__small_business_checking_or_share_draft","createdAt":1595896639332189957,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__special_indirect_vehicle_loan","createdAt":1595896641880199490,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__student_education_loan","createdAt":1595896639551558423,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira","createdAt":1595896643372890708,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira_cd_or_share_certificate","createdAt":1595896641621211665,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__trust","createdAt":1595896640709548779,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__vacation_club_holiday_savings","createdAt":1595896642256997553,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_recency__brokerage_account","createdAt":1595896641955724697,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_cd_or_share_certificate","createdAt":1595896642377298176,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_checking_or_share_draft","createdAt":1595896641640736297,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_credit_card","createdAt":1595896641199968081,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_indirect_auto_loan","createdAt":1595896643314414987,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_line_of_credit","createdAt":1595896641576267257,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_loan","createdAt":1595896641355763147,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_money_market","createdAt":1595896639970740640,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_new_auto_loan","createdAt":1595896642844117593,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_real_estate_loan","createdAt":1595896642953871689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_savings_or_share","createdAt":1595896643325231815,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_credit_card","createdAt":1595896642698454456,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_loan","createdAt":1595896641951407002,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_checking_or_share_draft","createdAt":1595896642711316173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_credit_card","createdAt":1595896642147545663,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_savings_or_share","createdAt":1595896642364919307,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_vehicle_loan","createdAt":1595896639876538030,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_used_auto_loan","createdAt":1595896643527381053,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_basic_checking_or_share_draft","createdAt":1595896639874545938,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_cd_or_share_certificate","createdAt":1595896643147292944,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_credit_card","createdAt":1595896642605033674,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_indirect_auto_loan","createdAt":1595896641097157897,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896642523867033,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_line_of_credit","createdAt":1595896641979726767,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_loan","createdAt":1595896639741458231,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_money_market","createdAt":1595896642350195677,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_new_auto_loan","createdAt":1595896642852653224,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_checking_or_share_draft","createdAt":1595896641391887646,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_savings_or_share","createdAt":1595896643501816261,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_real_estate_loan","createdAt":1595896643015250459,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_checking_or_share_draft","createdAt":1595896640413965217,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_savings_or_share","createdAt":1595896642440765022,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_savings_or_share","createdAt":1595896643523788186,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_credit_card","createdAt":1595896641228514181,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_loan","createdAt":1595896642815200189,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_checking_or_share_draft","createdAt":1595896639648900382,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_credit_card","createdAt":1595896639596071703,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_savings_or_share","createdAt":1595896642447703684,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_vehicle_loan","createdAt":1595896639863224318,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_used_auto_loan","createdAt":1595896639337315659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__debit_card","createdAt":1595896641091084110,"options":{"type":"int","base":0,"bitDepth":1,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__education_savings_or_share","createdAt":1595896643243122929,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__escrow_account","createdAt":1595896642288649376,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__health_savings_account","createdAt":1595896639713687734,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_line_of_credit","createdAt":1595896640644328270,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_loan","createdAt":1595896643005117660,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__insurance_product","createdAt":1595896640080338115,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__interest_only_legal_trust_account","createdAt":1595896640710544689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__membership_share","createdAt":1595896640920327282,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__other_service","createdAt":1595896641868683380,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_401k","createdAt":1595896643214447056,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_403b","createdAt":1595896639331510173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_529","createdAt":1595896641093053659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira","createdAt":1595896640406186395,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira_cd_or_share_certificate","createdAt":1595896642473602012,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__small_business_checking_or_share_draft","createdAt":1595896640631139451,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__special_indirect_vehicle_loan","createdAt":1595896641787887178,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__student_education_loan","createdAt":1595896640496522510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira","createdAt":1595896643383850510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira_cd_or_share_certificate","createdAt":1595896642145329304,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__trust","createdAt":1595896640506789797,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__vacation_club_holiday_savings","createdAt":1595896642206110046,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"stop_payment_recency","createdAt":1595896640954094741,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"survey_2053c210_adfc_4e14_afd3_9abc68a70719","createdAt":1595896642841258189,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2679bd7e_cdb8_4c24_a1da_e904799382eb","createdAt":1595896642498273724,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2f336120_7130_492b_b116_d93d560baa0a","createdAt":1595896639333711539,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_36e229ff_c0a6_48ad_928e_19b5abf69f11","createdAt":1595896642987004213,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa","createdAt":1595896643012024691,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796","createdAt":1595896640184709609,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_587ae933_b8e3_49a2_a23d_989df9ad119f","createdAt":1595896642862965911,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab","createdAt":1595896642292355510,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949","createdAt":1595896640788795677,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_9cebb459_162e_4518_88f6_bc1cdd630cd0","createdAt":1595896642343098690,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_a05de6d3_6620_4eb2_b9a2_511936680b57","createdAt":1595896643389135517,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_b570363c_ba94_4a7a_9891_141a1befc42b","createdAt":1595896643120229702,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_c358d6e0_4116_4230_84ed_869b315e89a1","createdAt":1595896642202109079,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_cce773ce_14e8_4fae_90cd_213a821a3bc0","createdAt":1595896640040535589,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_e49df858_4903_40ac_bd9f_96f764940f2c","createdAt":1595896642346035396,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9","createdAt":1595896640643626147,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"tablet_recency","createdAt":1595896640414042095,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"user_id","createdAt":1595896640846590883,"options":{"type":"int","base":0,"bitDepth":21,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"zip_code","createdAt":1595896640534832822,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}}],"shardWidth":1048576}]} diff --git a/cmd/sauron/sauron_test.go b/cmd/sauron/sauron_test.go new file mode 100644 index 000000000..53cf786de --- /dev/null +++ b/cmd/sauron/sauron_test.go @@ -0,0 +1,396 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strconv" + "testing" + + pb "github.com/golang/protobuf/proto" //nolint:staticcheck + pbuf "github.com/molecula/go-pilosa/v2/gopilosa_pbuf" + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/encoding/proto" + "github.com/pkg/errors" +) + +func TestSauron(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //remove the existing db file if present + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + + schemaJson := `{"indexes":[{"name":"scratch","createdAt":1611185870149882000,"options":{"keys":false,"trackExistence":true}, "fields":[{"name":"luminosity","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}}]}]}` + + client := &http.Client{} + t.Run("Schema", func(t *testing.T) { + _, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) + panicOn(err) + // verify that we can fetch it back + resp, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp.Body) + panicOn(err) + sbody := string(body) + if sbody[:62] != schemaJson[:62] { + fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) + panic("unexpected schema back") + } + }) + // add a set field "color" + //POST localhost:10101/index/scratch/field/color + t.Run("Create Non-Keyed SetField", func(t *testing.T) { + resp, err := client.Post(url+"/index/scratch/field/rating", "application/text", bytes.NewBuffer(nil)) + panicOn(err) + if resp.StatusCode != 200 { + panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) + } + }) + t.Run("Create Keyed SetField", func(t *testing.T) { + resp, err := client.Post(url+"/index/scratch/field/color", "application/text", bytes.NewBuffer([]byte(`{"options": {"keys": true}}`))) + panicOn(err) + if resp.StatusCode != 200 { + panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) + } + }) + t.Run("Create Time Field", func(t *testing.T) { + resp, err := client.Post(url+"/index/scratch/field/event", "application/text", bytes.NewBuffer([]byte(`{"options": { "type": "time", "timeQuantum": "YMDH" }}`))) + panicOn(err) + if resp.StatusCode != 200 { + panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) + } + }) + + // set some bits + + // set some bits + + t.Run("Set", func(t *testing.T) { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) + panicOn(err) + // test for dups + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) + panicOn(err) + // + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=red)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=green)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=green)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=green)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=green)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=yellow)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=yellow)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=yellow)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=yellow)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=orange)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=orange)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=orange)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=orange)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, luminosity=1)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, luminosity=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, luminosity=4)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, luminosity=5)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, luminosity=4)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, luminosity=3)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, luminosity=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, luminosity=1)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, rating=1)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(10, rating=1)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, rating=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, rating=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, rating=3)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, rating=4)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, rating=4)`)) + panicOn(err) + + //time fields + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T01:00)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T02:00)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, event=1,2021-02-05T02:05)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, event=1,2021-02-05T03:00)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, event=1,2021-02-05T04:00)`)) + panicOn(err) + }) + t.Run("Clear", func(t *testing.T) { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Clear(1, color=purple)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`ClearRow(color=purple)`)) + panicOn(err) + }) + t.Run("Store", func(t *testing.T) { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Store(Row(color=red), color=purple)`)) + panicOn(err) + }) + // verify that we can fetch it back + var schema pilosa.Schema + t.Run("Fetch Schema", func(t *testing.T) { + resp, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp.Body) + panicOn(err) + err = json.Unmarshal(body, &schema) + if err != nil { + panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) + } + }) + t.Run("Import Roaring", func(t *testing.T) { + //sets 12 bits in shard 0 + roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + idx := schema.Indexes[0] + var fld *pilosa.FieldInfo + for _, f := range idx.Fields { + if f.Name == "rating" { + fld = f + } + } + if fld == nil { + panic("field color not found") + } + msg := pilosa.ImportRoaringRequest{ + IndexCreatedAt: idx.CreatedAt, + FieldCreatedAt: fld.CreatedAt, + Clear: false, + Views: map[string][]byte{ + "": roaringData, + }, + UpdateExistence: true, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + furl := url + "/index/scratch/field/rating/import-roaring/0" + req, err := http.NewRequest("POST", furl, bytes.NewBuffer(data)) + panicOn(err) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + + // maybe need to Accept: "application/x-protobuf" + resp, err := client.Do(req) + panicOn(err) + _ = resp + }) + t.Run("Read Query", func(t *testing.T) { + for _, tst := range GetTests() { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(tst.Pql)) + panicOn(err) + } + }) +} + +func makeImportRequest(field *FieldInfo2, shard uint64, rows, columns []uint64, clear bool) (path string, data []byte, err error) { + msg := &pbuf.ImportRequest{ + Index: field.GetIndexName(), + IndexCreatedAt: field.GetIndexCreatedAt(), + Field: field.GetName(), + FieldCreatedAt: field.GetCreatedAt(), + Shard: shard, + RowIDs: rows, + ColumnIDs: columns, + } + data, err = pb.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling Import to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) + return path, data, nil +} +func makeImportKeyRequest(field *FieldInfo2, shard uint64, rows []string, columns []uint64, clear bool) (path string, data []byte, err error) { + msg := &pbuf.ImportRequest{ + Index: field.GetIndexName(), + IndexCreatedAt: field.GetIndexCreatedAt(), + Field: field.GetName(), + FieldCreatedAt: field.GetCreatedAt(), + Shard: shard, + RowKeys: rows, + ColumnIDs: columns, + } + data, err = pb.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling Import to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) + return path, data, nil +} + +func makeImportValueRequest(field *FieldInfo2, shard uint64, values []int64, columns []uint64, clear bool) (path string, data []byte, err error) { + msg := &pbuf.ImportValueRequest{ + Index: field.GetIndexName(), + IndexCreatedAt: field.GetIndexCreatedAt(), + Field: field.GetName(), + FieldCreatedAt: field.GetCreatedAt(), + Shard: shard, + Values: values, + ColumnIDs: columns, + } + data, err = pb.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling Import to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) + return path, data, nil +} +func TestSauronImport(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //remove the existing db file if present + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + schemaJson := `{ + "indexes": [{ + "name": "scratch", + "createdAt": 1611185870149882000, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [{ + "name": "bsi", + "createdAt": 1595896639332730413, + "options": { + "type": "int", + "base": 0, + "bitDepth": 31, + "min": -9223372036854775808, + "max": 9223372036854775807, + "keys": false, + "foreignIndex": "" + } + }, { + "name": "decimal", + "createdAt": 1613177295654228860, + "options": { + "type": "decimal", + "base": 0, + "scale": 1, + "bitDepth": 0, + "min": -922337203685477580.8, + "max": 922337203685477580.7, + "keys": false + } + },{ + "name": "setkeyfield", + "createdAt": 1613345793598357200, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + },{ + "name": "setfield", + "createdAt": 1613345793598357200, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }] + }] +}` + client := &http.Client{} + _, err = client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) + panicOn(err) + + t.Run("basic row/column", func(t *testing.T) { + f, err := proxy.sql.GetField("scratch", "setfield") + panicOn(err) + clear := false + cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} + rows := []uint64{0, 0, 0, 1, 1, 1, 2, 2, 2} + shard := uint64(0) + path, payload, err := makeImportRequest(f, shard, rows, cols, clear) + _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) + panicOn(err) + }) + t.Run("basic rowkey/column", func(t *testing.T) { + f, err := proxy.sql.GetField("scratch", "setkeyfield") + panicOn(err) + clear := false + cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} + rows := []string{"red", "red", "red", "blue", "blue", "blue", "green", "green", "green"} + shard := uint64(0) + path, payload, err := makeImportKeyRequest(f, shard, rows, cols, clear) + _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) + panicOn(err) + }) + t.Run("basic import values", func(t *testing.T) { + f, err := proxy.sql.GetField("scratch", "bsi") + panicOn(err) + clear := false + cols := []uint64{0, 1, 2, 3, 4} + values := []int64{40, 30, 20, 10, 5} + shard := uint64(0) + path, payload, err := makeImportValueRequest(f, shard, values, cols, clear) + _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) + panicOn(err) + }) + +} diff --git a/cmd/sauron/schema_test.go b/cmd/sauron/schema_test.go new file mode 100644 index 000000000..23f5e924b --- /dev/null +++ b/cmd/sauron/schema_test.go @@ -0,0 +1,353 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/pilosa/pilosa/v2" +) + +func NewSauronTestConfig(t *testing.T) *SauronConfig { + + bind := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) + bindGRPC := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) + gossipPort := fmt.Sprintf("%d", GetAvailPort()) + dataDir, err := ioutil.TempDir("", "pilosa-sauron-target-*") + panicOn(err) + + return &SauronConfig{ + DatabaseName: strings.ToLower(t.Name()), + Bind: bind, + BindGRPC: bindGRPC, + GossipPort: gossipPort, + DataDir: dataDir, + } +} + +func TestProxyCanHandlePostSchema(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //start with + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + + // prep for test by cleaning up leftovers from any old run. + //panicOn(proxy.sql.DropTablesWithPrefix(t.Name())) + + //schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } }, { "name ": "color", "options ": {} } ] }] }` + /* + schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } } ] }] }` + + + vv("Posting err = '%v'", schemaJson) + resp, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) + if resp.StatusCode != http.StatusNoContent { + panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) + } + // verify that we can fetch it back + resp2, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp2.Body) + panicOn(err) + sbody := string(body) + vv("body = '%v'", sbody) + if sbody[:62] != schemaJson[:62] { + fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) + panic("unxpected schema back") + } + */ + // load large schema too + client := &http.Client{} + schm, err := ioutil.ReadFile("sample_schema.json") + panicOn(err) + + resp, err := client.Post(url+"/schema", "application/json", bytes.NewBuffer(schm)) + panicOn(err) + if resp.StatusCode != http.StatusNoContent { + panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) + } + + // verify all tables expected are present + obs := make(map[string]bool) + for i, table := range proxy.sql.ListIndexFields() { + _ = i + obs[table] = true + if !expectedTables[table] { + panic(fmt.Sprintf("observed table '%v' but was not expected", table)) + } + } + for table := range expectedTables { + if !obs[table] { + panic(fmt.Sprintf("expected table '%v' but was not observed", table)) + } + } + +} + +func TestProxyCanCreateIndexFieldViaPost(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //start with + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + client := &http.Client{} + //create non-keyed or "simple" index + resp, err := client.Post(url+"/index/simple", "application/json", bytes.NewBuffer(nil)) + panicOn(err) + if resp.StatusCode != http.StatusOK { + panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) + } + + fields := []struct { + name string + options []byte + }{ + {name: "set", options: []byte{}}, + {name: "keyset", options: []byte{}}, + {name: "bsi", options: []byte{}}, + } + for _, field := range fields { + //create set field + resp, err = client.Post(fmt.Sprintf("%v/index/simple/field/%v", url, field.name), "application/json", bytes.NewBuffer(field.options)) + panicOn(err) + if resp.StatusCode != http.StatusOK { + panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) + } + } + // verify that we can fetch it back + resp2, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp2.Body) + panicOn(err) + var schema pilosa.Schema + err = json.Unmarshal(body, &schema) + if err != nil { + panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) + } + + expected := map[string]map[string]bool{ + "simple": {"bsi": true, "set": true, "keyset": true}, + } + for _, i := range schema.Indexes { + ll, ok := expected[i.Name] + if !ok { + panic(fmt.Sprintf("expected index not present: %v", i.Name)) + } + for _, f := range i.Fields { + _, ok := ll[f.Name] + if !ok { + panic(fmt.Sprintf("expected field '%v' not present in index'%v'", f.Name, i.Name)) + } + } + } +} + +var expectedTables = map[string]bool{ + "trait_store/aba": true, + "trait_store/ach_batch_recency": true, + "trait_store/ach_pass_thru_recency": true, + "trait_store/ach_payment_recency": true, + "trait_store/bools": true, + "trait_store/bools-exists": true, + "trait_store/central_group": true, + "trait_store/custom_audiences": true, + "trait_store/days_since_last_logon": true, + "trait_store/db": true, + "trait_store/desktop_recency": true, + "trait_store/domestic_wire_recency": true, + "trait_store/external_transfer_recency": true, + "trait_store/fields": true, + "trait_store/funds_transfer_recency": true, + "trait_store/international_wire_recency": true, + "trait_store/mobile_remote_deposit_recency": true, + "trait_store/payroll_recency": true, + "trait_store/pfm_category_total_current_balance__401k_investment": true, + "trait_store/pfm_category_total_current_balance__403b_investment": true, + "trait_store/pfm_category_total_current_balance__529_investment": true, + "trait_store/pfm_category_total_current_balance__any": true, + "trait_store/pfm_category_total_current_balance__auto_loan": true, + "trait_store/pfm_category_total_current_balance__brokerage_account": true, + "trait_store/pfm_category_total_current_balance__certificate_of_deposit": true, + "trait_store/pfm_category_total_current_balance__checking": true, + "trait_store/pfm_category_total_current_balance__credit_card": true, + "trait_store/pfm_category_total_current_balance__home_equity_loan": true, + "trait_store/pfm_category_total_current_balance__ira_investment": true, + "trait_store/pfm_category_total_current_balance__line_of_credit": true, + "trait_store/pfm_category_total_current_balance__loan": true, + "trait_store/pfm_category_total_current_balance__money_market": true, + "trait_store/pfm_category_total_current_balance__mortgage": true, + "trait_store/pfm_category_total_current_balance__personal_loan": true, + "trait_store/pfm_category_total_current_balance__roth_ira_investment": true, + "trait_store/pfm_category_total_current_balance__savings": true, + "trait_store/pfm_category_total_current_balance__simple_ira": true, + "trait_store/pfm_category_total_current_balance__student_loan": true, + "trait_store/pfm_category_total_current_balance__taxable_investment": true, + "trait_store/phone_recency": true, + "trait_store/product_count__brokerage_account": true, + "trait_store/product_count__commercial_cd_or_share_certificate": true, + "trait_store/product_count__commercial_checking_or_share_draft": true, + "trait_store/product_count__commercial_credit_card": true, + "trait_store/product_count__commercial_indirect_auto_loan": true, + "trait_store/product_count__commercial_line_of_credit": true, + "trait_store/product_count__commercial_loan": true, + "trait_store/product_count__commercial_money_market": true, + "trait_store/product_count__commercial_new_auto_loan": true, + "trait_store/product_count__commercial_real_estate_loan": true, + "trait_store/product_count__commercial_savings_or_share": true, + "trait_store/product_count__commercial_secured_credit_card": true, + "trait_store/product_count__commercial_secured_loan": true, + "trait_store/product_count__commercial_special_checking_or_share_draft": true, + "trait_store/product_count__commercial_special_credit_card": true, + "trait_store/product_count__commercial_special_savings_or_share": true, + "trait_store/product_count__commercial_special_vehicle_loan": true, + "trait_store/product_count__commercial_used_auto_loan": true, + "trait_store/product_count__consumer_basic_checking_or_share_draft": true, + "trait_store/product_count__consumer_cd_or_share_certificate": true, + "trait_store/product_count__consumer_credit_card": true, + "trait_store/product_count__consumer_indirect_auto_loan": true, + "trait_store/product_count__consumer_interest_bearing_checking_or_share_draft": true, + "trait_store/product_count__consumer_line_of_credit": true, + "trait_store/product_count__consumer_loan": true, + "trait_store/product_count__consumer_money_market": true, + "trait_store/product_count__consumer_new_auto_loan": true, + "trait_store/product_count__consumer_premium_checking_or_share_draft": true, + "trait_store/product_count__consumer_premium_savings_or_share": true, + "trait_store/product_count__consumer_real_estate_loan": true, + "trait_store/product_count__consumer_restricted_checking_or_share_draft": true, + "trait_store/product_count__consumer_restricted_savings_or_share": true, + "trait_store/product_count__consumer_savings_or_share": true, + "trait_store/product_count__consumer_secured_credit_card": true, + "trait_store/product_count__consumer_secured_loan": true, + "trait_store/product_count__consumer_special_checking_or_share_draft": true, + "trait_store/product_count__consumer_special_credit_card": true, + "trait_store/product_count__consumer_special_savings_or_share": true, + "trait_store/product_count__consumer_special_vehicle_loan": true, + "trait_store/product_count__consumer_used_auto_loan": true, + "trait_store/product_count__debit_card": true, + "trait_store/product_count__education_savings_or_share": true, + "trait_store/product_count__escrow_account": true, + "trait_store/product_count__health_savings_account": true, + "trait_store/product_count__home_equity_line_of_credit": true, + "trait_store/product_count__home_equity_loan": true, + "trait_store/product_count__insurance_product": true, + "trait_store/product_count__interest_only_legal_trust_account": true, + "trait_store/product_count__membership_share": true, + "trait_store/product_count__other_service": true, + "trait_store/product_count__plan_401k": true, + "trait_store/product_count__plan_403b": true, + "trait_store/product_count__plan_529": true, + "trait_store/product_count__roth_ira": true, + "trait_store/product_count__roth_ira_cd_or_share_certificate": true, + "trait_store/product_count__small_business_checking_or_share_draft": true, + "trait_store/product_count__special_indirect_vehicle_loan": true, + "trait_store/product_count__student_education_loan": true, + "trait_store/product_count__traditional_ira": true, + "trait_store/product_count__traditional_ira_cd_or_share_certificate": true, + "trait_store/product_count__trust": true, + "trait_store/product_count__vacation_club_holiday_savings": true, + "trait_store/product_recency__brokerage_account": true, + "trait_store/product_recency__commercial_cd_or_share_certificate": true, + "trait_store/product_recency__commercial_checking_or_share_draft": true, + "trait_store/product_recency__commercial_credit_card": true, + "trait_store/product_recency__commercial_indirect_auto_loan": true, + "trait_store/product_recency__commercial_line_of_credit": true, + "trait_store/product_recency__commercial_loan": true, + "trait_store/product_recency__commercial_money_market": true, + "trait_store/product_recency__commercial_new_auto_loan": true, + "trait_store/product_recency__commercial_real_estate_loan": true, + "trait_store/product_recency__commercial_savings_or_share": true, + "trait_store/product_recency__commercial_secured_credit_card": true, + "trait_store/product_recency__commercial_secured_loan": true, + "trait_store/product_recency__commercial_special_checking_or_share_draft": true, + "trait_store/product_recency__commercial_special_credit_card": true, + "trait_store/product_recency__commercial_special_savings_or_share": true, + "trait_store/product_recency__commercial_special_vehicle_loan": true, + "trait_store/product_recency__commercial_used_auto_loan": true, + "trait_store/product_recency__consumer_basic_checking_or_share_draft": true, + "trait_store/product_recency__consumer_cd_or_share_certificate": true, + "trait_store/product_recency__consumer_credit_card": true, + "trait_store/product_recency__consumer_indirect_auto_loan": true, + "trait_store/product_recency__consumer_interest_bearing_checking_or_share_draft": true, + "trait_store/product_recency__consumer_line_of_credit": true, + "trait_store/product_recency__consumer_loan": true, + "trait_store/product_recency__consumer_money_market": true, + "trait_store/product_recency__consumer_new_auto_loan": true, + "trait_store/product_recency__consumer_premium_checking_or_share_draft": true, + "trait_store/product_recency__consumer_premium_savings_or_share": true, + "trait_store/product_recency__consumer_real_estate_loan": true, + "trait_store/product_recency__consumer_restricted_checking_or_share_draft": true, + "trait_store/product_recency__consumer_restricted_savings_or_share": true, + "trait_store/product_recency__consumer_savings_or_share": true, + "trait_store/product_recency__consumer_secured_credit_card": true, + "trait_store/product_recency__consumer_secured_loan": true, + "trait_store/product_recency__consumer_special_checking_or_share_draft": true, + "trait_store/product_recency__consumer_special_credit_card": true, + "trait_store/product_recency__consumer_special_savings_or_share": true, + "trait_store/product_recency__consumer_special_vehicle_loan": true, + "trait_store/product_recency__consumer_used_auto_loan": true, + "trait_store/product_recency__debit_card": true, + "trait_store/product_recency__education_savings_or_share": true, + "trait_store/product_recency__escrow_account": true, + "trait_store/product_recency__health_savings_account": true, + "trait_store/product_recency__home_equity_line_of_credit": true, + "trait_store/product_recency__home_equity_loan": true, + "trait_store/product_recency__insurance_product": true, + "trait_store/product_recency__interest_only_legal_trust_account": true, + "trait_store/product_recency__membership_share": true, + "trait_store/product_recency__other_service": true, + "trait_store/product_recency__plan_401k": true, + "trait_store/product_recency__plan_403b": true, + "trait_store/product_recency__plan_529": true, + "trait_store/product_recency__roth_ira": true, + "trait_store/product_recency__roth_ira_cd_or_share_certificate": true, + "trait_store/product_recency__small_business_checking_or_share_draft": true, + "trait_store/product_recency__special_indirect_vehicle_loan": true, + "trait_store/product_recency__student_education_loan": true, + "trait_store/product_recency__traditional_ira": true, + "trait_store/product_recency__traditional_ira_cd_or_share_certificate": true, + "trait_store/product_recency__trust": true, + "trait_store/product_recency__vacation_club_holiday_savings": true, + "trait_store/stop_payment_recency": true, + "trait_store/survey_2053c210_adfc_4e14_afd3_9abc68a70719": true, + "trait_store/survey_2679bd7e_cdb8_4c24_a1da_e904799382eb": true, + "trait_store/survey_2f336120_7130_492b_b116_d93d560baa0a": true, + "trait_store/survey_36e229ff_c0a6_48ad_928e_19b5abf69f11": true, + "trait_store/survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa": true, + "trait_store/survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796": true, + "trait_store/survey_587ae933_b8e3_49a2_a23d_989df9ad119f": true, + "trait_store/survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab": true, + "trait_store/survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949": true, + "trait_store/survey_9cebb459_162e_4518_88f6_bc1cdd630cd0": true, + "trait_store/survey_a05de6d3_6620_4eb2_b9a2_511936680b57": true, + "trait_store/survey_b570363c_ba94_4a7a_9891_141a1befc42b": true, + "trait_store/survey_c358d6e0_4116_4230_84ed_869b315e89a1": true, + "trait_store/survey_cce773ce_14e8_4fae_90cd_213a821a3bc0": true, + "trait_store/survey_e49df858_4903_40ac_bd9f_96f764940f2c": true, + "trait_store/survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9": true, + "trait_store/tablet_recency": true, + "trait_store/user_id": true, + "trait_store/zip_code": true, +} diff --git a/cmd/sauron/sqlgen_test.go b/cmd/sauron/sqlgen_test.go new file mode 100644 index 000000000..b9d11a728 --- /dev/null +++ b/cmd/sauron/sqlgen_test.go @@ -0,0 +1,260 @@ +package main + +import ( + "strings" + "testing" + + "github.com/pilosa/pilosa/v2/pql" + "github.com/shurcooL/go-goon" +) + +/* +sqlite db this test validated this against +create table bits (field ,row , column , timestamp); + create table λbsi (field string,column bigint , val bigint); + CREATE VIEW columns as + select distinct column from λbits + UNION + select distinct column from λbsi; + + INSERT INTO λbits VALUES( 'color','red',1); + INSERT INTO λbits VALUES( 'color','red',2); + INSERT INTO λbits VALUES( 'color','red',3); + INSERT INTO λbits VALUES( 'color','red',4); + INSERT INTO λbits VALUES( 'color','red',5); + INSERT INTO λbits VALUES( 'color','green',2); + INSERT INTO λbits VALUES( 'color','green',4); + INSERT INTO λbits VALUES( 'color','green',6); + INSERT INTO λbits VALUES( 'color','green',8); + INSERT INTO λbits VALUES( 'color','yellow',1); + INSERT INTO λbits VALUES( 'color','yellow',3); + INSERT INTO λbits VALUES( 'color','yellow',5); + INSERT INTO λbits VALUES( 'color','yellow',7); + INSERT INTO λbits VALUES( 'color','orange',5); + INSERT INTO λbits VALUES( 'color','orange',6); + INSERT INTO λbits VALUES( 'color','orange',7); + INSERT INTO λbits VALUES( 'color','orange',8); + + INSERT INTO λbits VALUES( 'rating',1,8); + INSERT INTO λbits VALUES( 'rating',1,7); + INSERT INTO λbits VALUES( 'rating',1,10); + INSERT INTO λbits VALUES( 'rating',2,6); + INSERT INTO λbits VALUES( 'rating',2,5); + INSERT INTO λbits VALUES( 'rating',3,4); + INSERT INTO λbits VALUES( 'rating',4,3); + INSERT INTO λbits VALUES( 'rating',4,2); + + INSERT INTO λbsi VALUES( 'luminosity',1, 1); + INSERT INTO λbsi VALUES( 'luminosity',2, 2); + INSERT INTO λbsi VALUES( 'luminosity',3, 4); + INSERT INTO λbsi VALUES( 'luminosity',4, 5); + INSERT INTO λbsi VALUES( 'luminosity',5, 4); + INSERT INTO λbsi VALUES( 'luminosity',6, 3); + INSERT INTO λbsi VALUES( 'luminosity',7, 2); + INSERT INTO λbsi VALUES( 'luminosity',8, 1); +*/ +func GetTests() []struct { + Pql string + Sql string +} { + return []struct { + Pql string + Sql string + }{ + { + Pql: "Row(rating=0)", + Sql: `select distinct column from λbits where field="rating" AND row="0"`, + }, + { + Pql: "Row(color=red)", + Sql: `select distinct column from λbits where field="color" AND row="red"`, + }, + { + Pql: "Row(luminosity>2)", + Sql: `select distinct column from λbsi where field="luminosity" and val>2`, + }, + { + Pql: "Row(luminosity<2)", + Sql: `select distinct column from λbsi where field="luminosity" and val<2`, + }, + { + Pql: "Row(luminosity==4)", + Sql: `select distinct column from λbsi where field="luminosity" and val=4`, + }, + { + Pql: "Row(luminosity<=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val<=4`, + }, + { + Pql: "Row(luminosity>=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val>=4`, + }, + { + Pql: "Row(luminosity!=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val!=4`, + }, + { + Pql: "Row(2<=luminosity<=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val>=2 and val<=4`, + }, + { + Pql: "Row(22 and val<=4`, + }, + { + Pql: "Row(22 and val<4`, + }, + { + Pql: `Row(event=1, from='2021-02-01T00:00', to='2021-02-05T03:00')`, + Sql: `select distinct column from λbits where field="event" AND row="1" AND timestamp >= "2021-02-01 00:00" AND timestamp < "2021-02-05 03:00"`, + }, + { + Pql: "Count(row(color=red))", + Sql: `select count(*) from( select distinct column from λbits where field="color" AND row="red" )`, + }, + { + Pql: `Intersect(Row(color=red),Row(color=green))`, + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: "Count(Union(Row(color=red),Row(color=green)))", + Sql: `select count(*) from( select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") )`, + }, + { + Pql: "Union(Row(color=red),Row(color=green))", + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: "Difference(Row(color=red),Row(color=green))", + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" except select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: `All()`, + Sql: `select column from λcolumns`, + }, + { + Pql: "Not(Row(color=red))", + Sql: `select column from λcolumns except select distinct column from λbits where field="color" AND row="red"`, + }, + { + Pql: "Xor(Row(color=red),Row(color=green))", + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") except select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: "Distinct(field=luminosity)", + Sql: `select distinct val from λbsi where field="luminosity"`, + }, + { + Pql: "Distinct(Row(color=red),field=luminosity)", + Sql: `select distinct val from λbsi where field="luminosity" AND column in (select distinct column from λbits where field="color" AND row="red")`, + }, + { + Pql: "Distinct(Intersect(Row(color=red),Row(color=green)),field=luminosity)", + Sql: `select distinct val from λbsi where field="luminosity" AND column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green"))`, + }, + { + Pql: "Rows(color)", + Sql: `select distinct field, row from λbits where field="color" order by row`, + }, + { + Pql: "Rows(color, limit=10)", + Sql: `select distinct field, row from λbits where field="color" order by row limit 10`, + }, + { + Pql: "Rows(color, column=1)", + Sql: `select distinct field, row from λbits where field="color" and column='1' order by row`, + }, + // { // TODO (twg) Not supported by this test, can't reliably get key ordering + // Pql: "Rows(color, previous=green )", + // Sql: `select distinct field, row from λbits where field="color" and row>'green' order by row`, + // }, + { + Pql: "GroupBy(Rows(color))", + Sql: `select f1.field, f1.row,count(*) as cnt from λbits f1 where f1.field='color' group by f1.field,f1.row order by cnt desc`, + }, + { + Pql: "GroupBy(Rows(color),Rows(rating))", + Sql: `select f1.field, f1.row,f2.field, f2.row,count(*) as cnt from λbits f1 inner join λbits f2 on f1.column = f2.column where f1.field='color'and f2.field='rating' group by f1.field,f1.row,f2.field,f2.row order by cnt desc`, + }, + { + Pql: "TopN(color)", + Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, + }, + { + Pql: "TopN(color,n=2)", + Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc limit 2`, + }, + { + Pql: "TopN(color,Row(color=red),n=2)", + Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="color" AND row="red") group by row order by cnt desc limit 2`, + }, + { + Pql: "Topk(color)", + Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, + }, + { + Pql: "TopK(color, k=2, filter=Row(rating=1))", + Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="rating" AND row="1") group by row order by cnt desc limit 2`, + }, + { + Pql: `Min(field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity")`, + }, + { + Pql: `Max(field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select max(val) from λbsi where field="luminosity")`, + }, + { + Pql: `Min(Row(color=orange),field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select distinct column from λbits where field="color" AND row="orange")) and column in (select distinct column from λbits where field="color" AND row="orange")`, + }, + { + Pql: `Min(Intersect(Row(color=red),Row(color=orange)),field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))) and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))`, + }, + { + Pql: `Sum(field="luminosity")`, + Sql: `select sum(val),count(*) from λbsi where field="luminosity"`, + }, + { + Pql: `ConstRow(columns=[1,3,5])`, + Sql: `select column1 as column from (values (1),(3),(5))`, + }, + { + Pql: `UnionRows(Rows(color))`, + Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, + }, + { + Pql: `UnionRows(Rows(color),Rows(rating))`, + Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, + }, + { + Pql: `IncludesColumn(Row(color=red), column=1)`, + Sql: `select count(*)>0 from (select column from(select distinct column from λbits where field="color" AND row="red") where column='1')`, + }, + { + Pql: `Extract(Row(color=red),Rows(color),Rows(rating))`, + Sql: `select sq.field, sq.row, b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row where b.column in (select distinct column from λbits where field="color" AND row="red")`, + }, + { + Pql: `Extract(ConstRow(columns=[1,2,3]))`, + Sql: `select '' as field, '' as row,column from (select column1 as column from (values (1),(2),(3)))`, + }, + } +} +func TestParse(t *testing.T) { + + for _, tst := range GetTests() { + + q, err := pql.NewParser(strings.NewReader(tst.Pql)).Parse() + panicOn(err) + sql, _, _, err := ToSql(q.Calls, "") + panicOn(err) + if sql != tst.Sql { + vv("\npql:%v\ngenerated:\n%v\nexpected:\n%v\n", tst.Pql, sql, tst.Sql) + goon.Dump(q) + } + + } +} diff --git a/cmd/sauron/test_sqlite.sh b/cmd/sauron/test_sqlite.sh new file mode 100755 index 000000000..d0ab85b2f --- /dev/null +++ b/cmd/sauron/test_sqlite.sh @@ -0,0 +1,51 @@ +#!/bin/bash +db="poc2.db" +sqlite3 ${db} < 0 { + frames := runtime.CallersFrames(pc[:n]) + for i := 0; i <= target; i++ { + contender, more := frames.Next() + if i == target { + f = contender + } + if !more { + break + } + } + } + return f.Function +} From 6219b4ca8b9cf84c66decdacc61dc1becca29b28 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Feb 2021 17:47:28 -0600 Subject: [PATCH 02/11] add optional UpdateExistence on importRoaring --- api.go | 42 +++++-- encoding/proto/proto.go | 14 ++- handler.go | 13 +- http/client_test.go | 44 +++++++ internal/public.pb.go | 265 +++++++++++++++++++++++----------------- internal/public.proto | 1 + pilosa_internal_test.go | 45 ++++++- roaring/roaring.go | 41 +++++++ 8 files changed, 325 insertions(+), 140 deletions(-) diff --git a/api.go b/api.go index 55ea78af4..224c6bf15 100644 --- a/api.go +++ b/api.go @@ -17,6 +17,7 @@ package pilosa import ( + "bytes" "context" "encoding/binary" "encoding/csv" @@ -421,22 +422,25 @@ func importWorker(importWork chan importJob) { fallthrough case RequestActionSet: fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) - if fileMagic == roaring.MagicNumber { // if pilosa roaring format - err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear) - if err != nil { - return errors.Wrap(err, "importing pilosa roaring") - } - } else { + data := viewData + if fileMagic != roaring.MagicNumber { // if pilosa roaring format // must make a copy of data to operate on locally on standard roaring format. // field.importRoaring changes the standard roaring run format to pilosa roaring - data := make([]byte, len(viewData)) + data = make([]byte, len(viewData)) copy(data, viewData) - err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) - - if err != nil { - return errors.Wrap(err, "importing standard roaring") + } + if j.req.UpdateExistence { + if ef := j.field.idx.existenceField(); ef != nil { + existence := combineForExistence(data) + ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false) } } + + err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) + + if err != nil { + return errors.Wrap(err, "importing standard roaring") + } } return nil }(); err != nil { @@ -453,6 +457,22 @@ func importWorker(importWork chan importJob) { } } +// merge all rows to singled existence row +func combineForExistence(inputRoaringData []byte) []byte { + rowSize := uint64(1 << shardVsContainerExponent) + rit, err := roaring.NewRoaringIterator(inputRoaringData) + if err != nil { + panicOn(err) + } + bm := roaring.NewBitmap() + bm.MergeRoaringRawIteratorIntoExists(rit, rowSize) + buf := new(bytes.Buffer) + + _, err = bm.WriteTo(buf) + panicOn(err) + return buf.Bytes() +} + // ImportRoaring is a low level interface for importing data to Pilosa when // extremely high throughput is desired. The data must be encoded in a // particular way which may be unintuitive (discussed below). The data is merged diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 2091cc850..c67962223 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -462,12 +462,13 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) * i++ } return &internal.ImportRoaringRequest{ - IndexCreatedAt: m.IndexCreatedAt, - FieldCreatedAt: m.FieldCreatedAt, - Clear: m.Clear, - Action: m.Action, - Block: uint64(m.Block), - Views: views, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Clear: m.Clear, + Action: m.Action, + Block: uint64(m.Block), + Views: views, + UpdateExistence: m.UpdateExistence, } } @@ -1256,6 +1257,7 @@ func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest m.Views = views m.IndexCreatedAt = pb.IndexCreatedAt m.FieldCreatedAt = pb.FieldCreatedAt + m.UpdateExistence = pb.UpdateExistence } func (s Serializer) decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) { diff --git a/handler.go b/handler.go index b50e6e946..ca5b119f5 100644 --- a/handler.go +++ b/handler.go @@ -243,12 +243,13 @@ const ( // ImportRoaringRequest describes the import request structure // for an import containing roaring-encoded data. type ImportRoaringRequest struct { - IndexCreatedAt int64 - FieldCreatedAt int64 - Clear bool - Action string // [set, clear, overwrite] - Block int - Views map[string][]byte + IndexCreatedAt int64 + FieldCreatedAt int64 + Clear bool + Action string // [set, clear, overwrite] + Block int + Views map[string][]byte + UpdateExistence bool } // ValidateWithTimestamp ensures that the payload of the request is valid. diff --git a/http/client_test.go b/http/client_test.go index b1b154647..87a010501 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -1435,3 +1435,47 @@ func TestClient_ServerInfoHasTxSrc(t *testing.T) { } pilosa.MustTxsrcToTxtype(si.TxSrc) // panics if invalid } +func TestClient_ImportRoaringExists(t *testing.T) { + cluster := test.MustNewCluster(t, 1) + err := cluster.Start() + if err != nil { + t.Fatalf("starting cluster: %v", err) + } + defer cluster.Close() + + node := cluster.GetNode(0) + _, err = node.API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = node.API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + // Send import request. + host := node.URL() + c := MustNewClient(host, http.GetHTTPClient(nil)) + // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] + roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") + + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { + t.Fatal(err) + } + qr, err := node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + got := qr.Results[0].(*pilosa.Row).Columns() + if !reflect.DeepEqual(got, []uint64{}) { + t.Fatalf(" Row unexpected columns: got %+v expected: %+v", got, []uint64{}) + } + roaringReq.UpdateExistence = true + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { + t.Fatal(err) + } + + expected := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537} + qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + got = qr.Results[0].(*pilosa.Row).Columns() + if !reflect.DeepEqual(got, expected) { + t.Fatalf("All unexpected columns: got %+v expected: %+v", got, expected) + } + +} diff --git a/internal/public.pb.go b/internal/public.pb.go index e406e06ab..109e68c08 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -2495,6 +2495,7 @@ type ImportRoaringRequest struct { Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` IndexCreatedAt int64 `protobuf:"varint,5,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` FieldCreatedAt int64 `protobuf:"varint,6,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` + UpdateExistence bool `protobuf:"varint,7,opt,name=UpdateExistence,proto3" json:"UpdateExistence,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2575,6 +2576,13 @@ func (m *ImportRoaringRequest) GetFieldCreatedAt() int64 { return 0 } +func (m *ImportRoaringRequest) GetUpdateExistence() bool { + if m != nil { + return m.UpdateExistence + } + return false +} + type ImportColumnAttrsRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Shard int64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` @@ -2761,117 +2769,119 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1757 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0xe4, 0x48, - 0x15, 0x8f, 0xdb, 0xee, 0x7f, 0xaf, 0x3b, 0x99, 0x6c, 0x4d, 0xcf, 0x62, 0x0d, 0x99, 0x6c, 0x63, - 0x05, 0xb6, 0x41, 0xab, 0xac, 0x12, 0x76, 0x60, 0x0e, 0xfc, 0xd9, 0x64, 0x3a, 0x4b, 0xac, 0x21, - 0xd9, 0xa1, 0x12, 0x65, 0xc5, 0x05, 0xc9, 0xe9, 0x2e, 0x7a, 0x2d, 0xdc, 0xed, 0xc6, 0xed, 0xde, - 0x4e, 0x2e, 0x48, 0x7c, 0x86, 0xbd, 0x70, 0x43, 0xdc, 0xf8, 0x1c, 0x5c, 0xe0, 0xc8, 0x11, 0x89, - 0x0b, 0x1a, 0xb8, 0xf2, 0x1d, 0xd0, 0x7b, 0xe5, 0x72, 0x95, 0xdd, 0x4e, 0x26, 0x1a, 0x71, 0xab, - 0xf7, 0xa7, 0x5e, 0xd5, 0xfb, 0xbd, 0x57, 0xef, 0x3d, 0x1b, 0xba, 0xf3, 0xe5, 0x75, 0x14, 0x8e, - 0xf6, 0xe7, 0x49, 0x9c, 0xc6, 0xac, 0x15, 0xce, 0x52, 0x91, 0xcc, 0x82, 0xc8, 0xfb, 0xa3, 0x05, - 0x36, 0x8f, 0x57, 0xcc, 0x85, 0xe6, 0xcb, 0x38, 0x5a, 0x4e, 0x67, 0x0b, 0xd7, 0xea, 0xdb, 0x03, - 0x87, 0x2b, 0x92, 0x31, 0x70, 0x5e, 0x89, 0xdb, 0x85, 0x6b, 0xf7, 0xed, 0x41, 0x9b, 0xd3, 0x9a, - 0xed, 0x41, 0xfd, 0x28, 0x4d, 0x93, 0x85, 0x5b, 0xeb, 0xdb, 0x83, 0xce, 0xe1, 0xd6, 0xbe, 0xb2, - 0xb7, 0x8f, 0x6c, 0x2e, 0x85, 0x68, 0x93, 0xc7, 0x41, 0x12, 0xce, 0x26, 0xae, 0xd3, 0xb7, 0x06, - 0x5d, 0xae, 0x48, 0xd6, 0x83, 0xba, 0x3f, 0x1b, 0x8b, 0x1b, 0xb7, 0xde, 0xb7, 0x06, 0x6d, 0x2e, - 0x09, 0xe4, 0x7e, 0x16, 0x8a, 0x68, 0xec, 0x36, 0x24, 0x97, 0x08, 0x6f, 0x1f, 0xda, 0x3c, 0x5e, - 0x9d, 0x05, 0x69, 0x12, 0xde, 0xb0, 0x6f, 0x81, 0xc3, 0xe3, 0x95, 0xbc, 0x63, 0xe7, 0x70, 0x53, - 0x9f, 0xcb, 0xe3, 0x15, 0x27, 0x91, 0x77, 0x06, 0xed, 0x8b, 0x70, 0x32, 0x13, 0x63, 0x74, 0xeb, - 0x03, 0xb0, 0x5f, 0xc7, 0xa8, 0x6e, 0xad, 0xab, 0xa3, 0x04, 0x15, 0xce, 0xc5, 0xc4, 0xad, 0x55, - 0x2a, 0x9c, 0x8b, 0x89, 0xf7, 0x02, 0xb6, 0x78, 0xbc, 0xf2, 0xc7, 0x62, 0x96, 0x86, 0xbf, 0x0e, - 0x45, 0x42, 0x80, 0xe4, 0x77, 0x70, 0xe4, 0xa1, 0x39, 0x48, 0x35, 0x0d, 0x92, 0xf7, 0x14, 0x1a, - 0xfe, 0xf0, 0xe7, 0xe1, 0x22, 0x65, 0xdb, 0x60, 0xfb, 0x43, 0xb5, 0x01, 0x97, 0x9e, 0x0f, 0xef, - 0x9d, 0xdc, 0xa4, 0x49, 0x30, 0x4a, 0xc5, 0xd8, 0x1f, 0x4a, 0xa8, 0xd9, 0x16, 0xd4, 0xfc, 0x21, - 0xdd, 0xd5, 0xe1, 0x35, 0x7f, 0xc8, 0xf6, 0xc0, 0xb9, 0x0a, 0x22, 0x05, 0xf2, 0xb6, 0xbe, 0x9c, - 0x34, 0xcb, 0x49, 0xea, 0x5d, 0x17, 0x4c, 0x65, 0x38, 0xbd, 0x0f, 0x0d, 0x42, 0x4f, 0x1e, 0xda, - 0xe6, 0x19, 0xc5, 0x9e, 0xeb, 0x30, 0x4b, 0xab, 0xdf, 0xd4, 0x56, 0xd7, 0x2e, 0x94, 0xe7, 0x80, - 0xf7, 0x0c, 0x9a, 0xaf, 0xc4, 0x2d, 0xf9, 0xa2, 0x3c, 0xb5, 0x0c, 0x4f, 0xff, 0x69, 0xc1, 0xe3, - 0x7c, 0xf7, 0x65, 0x70, 0x1d, 0x89, 0xab, 0x20, 0x5a, 0x0a, 0xb6, 0xa7, 0xfc, 0xb6, 0xaa, 0xee, - 0x7f, 0xba, 0x41, 0x58, 0xb0, 0x0f, 0x73, 0xec, 0x50, 0xed, 0x3d, 0xad, 0x96, 0x1d, 0x79, 0xba, - 0x91, 0x65, 0xdd, 0x0e, 0xb4, 0x8e, 0x2f, 0x7c, 0x32, 0xed, 0xda, 0x7d, 0x6b, 0x60, 0x9f, 0x6e, - 0xf0, 0x9c, 0xc3, 0x9e, 0x42, 0xf3, 0x6c, 0x99, 0x8a, 0x1b, 0x7f, 0x48, 0xd9, 0xe6, 0x9c, 0x6e, - 0x70, 0xc5, 0xc0, 0x9d, 0xb4, 0x7c, 0x25, 0x6e, 0x65, 0xca, 0xe1, 0x4e, 0xc5, 0x61, 0x3d, 0x70, - 0x8e, 0xe3, 0x38, 0xa2, 0xb4, 0x6b, 0xe1, 0x69, 0x48, 0x1d, 0x37, 0xa1, 0x4e, 0x86, 0xbd, 0xdf, - 0x41, 0xaf, 0xe8, 0x5c, 0x16, 0x2e, 0x06, 0x36, 0xda, 0xb3, 0x32, 0x7b, 0x48, 0xb0, 0x6d, 0x0a, - 0x61, 0x2d, 0x3b, 0x1f, 0x83, 0xf8, 0x1c, 0x1a, 0x64, 0x46, 0x3e, 0xa0, 0xce, 0xe1, 0xb3, 0x0a, - 0xc0, 0x35, 0x64, 0x3c, 0x53, 0x3e, 0x6e, 0x13, 0xe2, 0x9f, 0x27, 0xfe, 0xd0, 0xfb, 0x71, 0x19, - 0x5c, 0x8a, 0x25, 0x06, 0xe2, 0x3c, 0x98, 0x0a, 0x79, 0x3e, 0xa7, 0x35, 0xf2, 0x2e, 0x6f, 0xe7, - 0x82, 0x2e, 0xd0, 0xe6, 0xb4, 0xf6, 0x7e, 0x6f, 0xc1, 0x56, 0x71, 0x3f, 0xde, 0xc9, 0xc8, 0x8e, - 0x7b, 0xee, 0x44, 0x5a, 0x79, 0xf2, 0xbc, 0x28, 0x27, 0xcf, 0xee, 0x5d, 0xfb, 0xca, 0xf9, 0xf3, - 0x13, 0x70, 0x5e, 0x07, 0x61, 0xb2, 0x96, 0xe1, 0xdb, 0x12, 0x42, 0x9b, 0xae, 0x6b, 0xcb, 0x58, - 0xd4, 0x5f, 0xc6, 0xcb, 0x59, 0x2a, 0x31, 0xe4, 0x92, 0xf0, 0x4e, 0xa0, 0x8d, 0xfb, 0xa5, 0xe3, - 0x9e, 0x34, 0x96, 0xa5, 0x95, 0x51, 0x7b, 0x90, 0xcb, 0xe5, 0x41, 0x79, 0x29, 0xa9, 0x99, 0xa5, - 0xe4, 0x14, 0x00, 0xa5, 0x0b, 0x69, 0x67, 0x0f, 0xea, 0x44, 0x65, 0x20, 0x94, 0x0d, 0x49, 0xe1, - 0x1d, 0x96, 0x9e, 0x61, 0x01, 0x4b, 0x7f, 0xf0, 0x09, 0x8a, 0x65, 0x42, 0xe2, 0x6d, 0x6c, 0x9e, - 0xa5, 0xcc, 0x12, 0x5a, 0x12, 0xba, 0x78, 0xa5, 0x0d, 0x58, 0x86, 0x01, 0xe4, 0x62, 0x59, 0x19, - 0x2a, 0x3f, 0x89, 0xc0, 0x67, 0xcb, 0xe3, 0x95, 0x86, 0x24, 0xa3, 0xd8, 0xb7, 0xd5, 0x29, 0x0e, - 0xf9, 0xfc, 0xc8, 0x78, 0x4a, 0x78, 0x0b, 0x75, 0xec, 0xaf, 0x00, 0x7e, 0x96, 0xc4, 0xcb, 0x39, - 0x81, 0xc6, 0x06, 0x50, 0x27, 0x2a, 0xf3, 0x8f, 0xe9, 0x4d, 0xea, 0x6e, 0x5c, 0x2a, 0x54, 0x83, - 0x8e, 0xc1, 0x39, 0x9a, 0x4c, 0xe4, 0x4b, 0xe3, 0xb8, 0xc4, 0x54, 0x6a, 0x5d, 0x05, 0x51, 0x2e, - 0xbe, 0x0a, 0xa2, 0xcc, 0x6f, 0x5c, 0x16, 0xcd, 0xd8, 0xca, 0xcc, 0x53, 0x68, 0x7d, 0x16, 0xc5, - 0x41, 0x8a, 0xca, 0x68, 0xcb, 0xe2, 0x39, 0xcd, 0x0e, 0x00, 0x86, 0x62, 0x14, 0x4e, 0x83, 0x08, - 0xa5, 0x4e, 0xb9, 0x00, 0x64, 0x32, 0x6e, 0x28, 0x79, 0xcf, 0xa1, 0x99, 0x51, 0xd5, 0xd8, 0x23, - 0xf7, 0x62, 0x14, 0x44, 0x42, 0xdd, 0x82, 0x08, 0xef, 0x0b, 0xd8, 0x94, 0xc9, 0x88, 0xad, 0xe9, - 0x42, 0xa4, 0x0f, 0x48, 0xc5, 0x07, 0x35, 0x39, 0xef, 0xcf, 0x16, 0x38, 0xb8, 0x52, 0x06, 0x2c, - 0x6d, 0xc0, 0x7c, 0x8d, 0x8e, 0x7c, 0x8d, 0xac, 0x0f, 0x9d, 0x8b, 0x14, 0x7b, 0xa0, 0x2e, 0x63, - 0x6d, 0x6e, 0xb2, 0x10, 0x2f, 0x7f, 0x96, 0xea, 0x70, 0xdb, 0x3c, 0xa7, 0xd9, 0x0e, 0xb4, 0xb1, - 0x36, 0x49, 0x21, 0x16, 0xb2, 0x16, 0xd7, 0x0c, 0xb6, 0x0b, 0xa0, 0x90, 0x5d, 0x0a, 0xaa, 0x66, - 0x16, 0x37, 0x38, 0xde, 0xc7, 0xd0, 0xc4, 0x9b, 0x9e, 0x05, 0x73, 0xed, 0x9b, 0x75, 0x9f, 0x6f, - 0x7f, 0xaa, 0x41, 0xf7, 0x17, 0x4b, 0x91, 0xdc, 0x72, 0xf1, 0xdb, 0xa5, 0x58, 0xa4, 0x88, 0x2d, - 0xd1, 0x2a, 0x97, 0x89, 0xc0, 0xac, 0xbd, 0xf8, 0x32, 0x48, 0xc6, 0x12, 0x29, 0x87, 0x67, 0x14, - 0xfa, 0xaa, 0x31, 0x5f, 0x90, 0xaf, 0x2d, 0x6e, 0xb2, 0x28, 0xdf, 0xc5, 0x34, 0x4e, 0x95, 0x33, - 0x19, 0xc5, 0x06, 0xf0, 0xe8, 0xe4, 0x66, 0x14, 0x2d, 0xc7, 0x82, 0xc7, 0x2b, 0xb9, 0x9b, 0x8a, - 0x33, 0x2f, 0xb3, 0xd9, 0x77, 0xb0, 0xb8, 0x11, 0x4b, 0x95, 0xa6, 0x26, 0x29, 0x96, 0xb8, 0xec, - 0x00, 0xba, 0x27, 0xd3, 0x6b, 0x31, 0x1e, 0x8b, 0xf1, 0x30, 0x48, 0x03, 0xb7, 0x55, 0x35, 0x40, - 0x14, 0x54, 0xd8, 0x1e, 0x6c, 0xbe, 0x4e, 0xc4, 0x65, 0x12, 0xcc, 0x16, 0x51, 0x90, 0x8a, 0xb1, - 0xdb, 0x26, 0xcb, 0x45, 0xa6, 0xf7, 0xb5, 0x05, 0x9b, 0x19, 0x46, 0x8b, 0x79, 0x3c, 0x5b, 0x08, - 0x4c, 0x84, 0x93, 0x24, 0x51, 0x89, 0x70, 0x92, 0x24, 0xec, 0x63, 0x68, 0x72, 0xb1, 0x58, 0x46, - 0xa9, 0xca, 0xa5, 0x27, 0xfa, 0x5c, 0xb5, 0x77, 0x19, 0xa5, 0x5c, 0x69, 0xb1, 0x9f, 0xc2, 0x56, - 0x21, 0x5b, 0x55, 0xf3, 0xf8, 0x86, 0xde, 0x57, 0x90, 0xf3, 0x92, 0xba, 0xf7, 0xdf, 0x3a, 0x74, - 0x0c, 0xcb, 0x79, 0x2a, 0x22, 0x8a, 0x9b, 0x59, 0x2a, 0x7e, 0x40, 0x93, 0xdf, 0x1d, 0xb3, 0x11, - 0x56, 0xae, 0x2e, 0x58, 0xe7, 0x59, 0xf2, 0x5a, 0xe7, 0xba, 0x5c, 0xda, 0xf7, 0x95, 0x4b, 0x9c, - 0x23, 0xbf, 0x0c, 0x66, 0x13, 0x31, 0xa6, 0xe4, 0x6d, 0x71, 0x45, 0xb2, 0x7d, 0x5d, 0x3b, 0x28, - 0xda, 0x85, 0x8a, 0xa4, 0x24, 0x5c, 0xd7, 0x17, 0x59, 0x0b, 0x71, 0x7e, 0x68, 0xca, 0xac, 0x92, - 0x14, 0xfb, 0x11, 0x6c, 0x7d, 0x1e, 0x8d, 0x75, 0x9d, 0x5b, 0x64, 0xb1, 0xec, 0x69, 0x6b, 0x5a, - 0xc8, 0x4b, 0xba, 0xec, 0xd3, 0xf2, 0x38, 0x47, 0x51, 0xed, 0x1c, 0xba, 0x05, 0xff, 0x0d, 0x39, - 0x2f, 0x8f, 0x7f, 0x07, 0xc6, 0x7c, 0xe9, 0x02, 0x6d, 0x7e, 0xac, 0x37, 0xe7, 0x22, 0x6e, 0x4c, - 0xa1, 0x9f, 0x98, 0x7d, 0xc7, 0xed, 0xd0, 0x9e, 0x5e, 0x11, 0x3f, 0x29, 0xe3, 0x66, 0x7f, 0x3a, - 0x30, 0x9a, 0x9e, 0xdb, 0x2d, 0x1f, 0x94, 0x8b, 0xb8, 0xd1, 0x1a, 0xfd, 0x8a, 0x59, 0xd0, 0xdd, - 0xa4, 0xad, 0xd5, 0x83, 0x9e, 0x54, 0xe1, 0x15, 0x13, 0xe4, 0xa7, 0xe5, 0xa9, 0xc1, 0xdd, 0x2a, - 0x03, 0x55, 0x94, 0xf3, 0xf2, 0x94, 0x71, 0x60, 0x0c, 0xee, 0xee, 0xa3, 0xf2, 0xfd, 0x73, 0x11, - 0x37, 0xc6, 0xfb, 0x1f, 0x42, 0xc7, 0x0c, 0xec, 0x36, 0x6d, 0x7a, 0x52, 0x15, 0xd8, 0x05, 0x37, - 0x35, 0xbd, 0xbf, 0xd6, 0x60, 0xd3, 0x9f, 0xce, 0xe3, 0x24, 0x35, 0x4a, 0x95, 0xfc, 0xc4, 0xb0, - 0x2a, 0x3f, 0x31, 0x6a, 0xa5, 0x66, 0x4c, 0x25, 0x8b, 0x4a, 0x94, 0xc3, 0x25, 0x61, 0x24, 0xa0, - 0x53, 0x48, 0xc0, 0x1d, 0x68, 0xcb, 0xd7, 0x86, 0xa2, 0x3a, 0x89, 0x34, 0x43, 0x7e, 0xf4, 0xac, - 0x68, 0xa0, 0x6d, 0xd2, 0x88, 0xac, 0x48, 0x2c, 0xcf, 0x52, 0x8d, 0x84, 0x2d, 0x12, 0x1a, 0x1c, - 0x94, 0x5f, 0x86, 0x53, 0xb1, 0x48, 0x83, 0xe9, 0x1c, 0xeb, 0x9d, 0x3d, 0xb0, 0xb9, 0xc1, 0xc1, - 0x52, 0x47, 0x4e, 0xbc, 0x4c, 0x04, 0x56, 0x9e, 0xa3, 0x94, 0x52, 0xd7, 0xe6, 0x25, 0x2e, 0xea, - 0x91, 0x5b, 0x5a, 0x0f, 0xa4, 0x5e, 0x91, 0x4b, 0xed, 0x3a, 0x12, 0x41, 0x42, 0x09, 0xd9, 0xe2, - 0x92, 0xf0, 0xfe, 0x51, 0x03, 0x26, 0x91, 0x94, 0x03, 0xe9, 0xff, 0x0d, 0xce, 0xfb, 0x61, 0x2b, - 0x82, 0xd3, 0x5c, 0x03, 0xe7, 0xfd, 0x7c, 0x8c, 0x96, 0xc0, 0x64, 0x14, 0xf6, 0x18, 0xdd, 0xe1, - 0x24, 0xaa, 0x16, 0x37, 0x59, 0xcc, 0x83, 0xae, 0xd1, 0x5e, 0xf1, 0xbd, 0xa3, 0xed, 0x02, 0xaf, - 0x02, 0x5a, 0x78, 0x20, 0xb4, 0x9d, 0xfb, 0xa1, 0xed, 0x9a, 0xd0, 0x7e, 0x6d, 0x41, 0xf7, 0x28, - 0x8d, 0xa7, 0xe1, 0x88, 0x8b, 0x51, 0x9c, 0x8c, 0xef, 0x06, 0x55, 0xc2, 0x57, 0x33, 0xe1, 0xdb, - 0x07, 0xdb, 0xff, 0x2a, 0xc9, 0x8a, 0xef, 0x8e, 0x31, 0x00, 0xae, 0xc5, 0x8a, 0xa3, 0x22, 0xfb, - 0x10, 0x6a, 0x7e, 0x42, 0x99, 0x5b, 0x68, 0x1b, 0x85, 0x47, 0xc2, 0x6b, 0x7e, 0xe2, 0x7d, 0x04, - 0x3d, 0x79, 0x29, 0x25, 0xca, 0xda, 0x58, 0x0f, 0xea, 0x27, 0x49, 0x12, 0xab, 0x46, 0x26, 0x09, - 0xef, 0x06, 0x7a, 0x79, 0xf3, 0xc3, 0xc0, 0xbc, 0x4b, 0x7e, 0x54, 0xfd, 0x51, 0xe8, 0x43, 0xe7, - 0x3c, 0x4e, 0xbf, 0x48, 0xc2, 0x94, 0x6a, 0x8d, 0xec, 0x1d, 0x26, 0xcb, 0xfb, 0x2e, 0x3c, 0x29, - 0x9d, 0xac, 0xfb, 0x2d, 0xa6, 0x94, 0xad, 0xbf, 0xae, 0x2f, 0xe0, 0x71, 0xae, 0xea, 0x0f, 0xdf, - 0xe9, 0x8e, 0xeb, 0x46, 0xbf, 0x67, 0x78, 0x4e, 0x46, 0xb3, 0xe3, 0x2b, 0xbc, 0xf1, 0x8e, 0xc1, - 0xcd, 0xd0, 0x94, 0x3f, 0x3c, 0xb2, 0x1b, 0x5c, 0x85, 0x62, 0x75, 0xd7, 0x77, 0x1b, 0x4d, 0x25, - 0x35, 0xfa, 0x4d, 0x42, 0x6b, 0xef, 0x3f, 0x16, 0xf4, 0xaa, 0x8c, 0xe8, 0xe4, 0xb2, 0x8c, 0xe4, - 0x62, 0x2f, 0xa0, 0xfe, 0x55, 0x28, 0x56, 0x6a, 0xc2, 0xf0, 0xd6, 0x42, 0xbe, 0x76, 0x13, 0x2e, - 0x37, 0xe0, 0xd3, 0x3a, 0x1a, 0xa5, 0x61, 0x3c, 0x53, 0x1f, 0x1d, 0x92, 0xc2, 0x73, 0x8e, 0xa3, - 0x78, 0xf4, 0x1b, 0xf9, 0x39, 0xcd, 0x25, 0x51, 0xf1, 0x54, 0xea, 0x0f, 0x7c, 0x2a, 0x8d, 0xaa, - 0xa7, 0xe2, 0xfd, 0xc5, 0x52, 0x58, 0x19, 0x83, 0xe1, 0x5b, 0x23, 0xa6, 0x1f, 0x88, 0xad, 0x1e, - 0x88, 0x2b, 0xa7, 0x5b, 0x3d, 0xc4, 0x2b, 0x12, 0x27, 0x6a, 0x5c, 0xd2, 0xbf, 0x14, 0x87, 0xa2, - 0x94, 0xd3, 0x6f, 0xa9, 0x4a, 0xeb, 0xce, 0x36, 0xaa, 0x9c, 0xf5, 0x7e, 0x59, 0xe8, 0x5b, 0x68, - 0xf4, 0x68, 0x32, 0x49, 0xc4, 0x24, 0x48, 0x55, 0x9c, 0x35, 0x83, 0x7d, 0x04, 0x0d, 0x52, 0x56, - 0xa1, 0xaa, 0x1e, 0x5c, 0x32, 0x9d, 0xe3, 0xed, 0xbf, 0xbd, 0xd9, 0xb5, 0xfe, 0xfe, 0x66, 0xd7, - 0xfa, 0xd7, 0x9b, 0x5d, 0xeb, 0x0f, 0xff, 0xde, 0xdd, 0xb8, 0x6e, 0xd0, 0x3f, 0xbc, 0xef, 0xff, - 0x2f, 0x00, 0x00, 0xff, 0xff, 0x0a, 0x79, 0xf3, 0x2b, 0xd3, 0x13, 0x00, 0x00, + // 1779 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0x23, 0x49, + 0x15, 0x4f, 0xbb, 0xdb, 0xb1, 0xfd, 0xec, 0x64, 0xb2, 0x35, 0x9e, 0xa5, 0x35, 0xcc, 0x64, 0x43, + 0x2b, 0xb0, 0x06, 0xad, 0xb2, 0xca, 0xb0, 0x03, 0x73, 0xe0, 0xcf, 0x26, 0xe3, 0x2c, 0x69, 0x0d, + 0x93, 0x1d, 0x2a, 0x43, 0x56, 0x5c, 0x90, 0x3a, 0x76, 0xe1, 0x6d, 0xd1, 0x76, 0x9b, 0x76, 0x79, + 0x9d, 0x5c, 0x90, 0xf8, 0x02, 0x5c, 0xf6, 0xc2, 0x0d, 0x71, 0xe3, 0x73, 0x70, 0x81, 0x23, 0x47, + 0x24, 0x2e, 0x68, 0x38, 0xf3, 0x1d, 0xd0, 0x7b, 0xd5, 0xd5, 0x55, 0xdd, 0xee, 0x64, 0xa3, 0xd1, + 0xde, 0xea, 0xfd, 0xa9, 0x57, 0xf5, 0x7e, 0xef, 0xd5, 0x7b, 0xaf, 0x1b, 0x7a, 0xf3, 0xe5, 0x65, + 0x12, 0x8f, 0x0e, 0xe6, 0x59, 0x2a, 0x53, 0xd6, 0x8e, 0x67, 0x52, 0x64, 0xb3, 0x28, 0x09, 0xfe, + 0xec, 0x80, 0xcb, 0xd3, 0x15, 0xf3, 0xa1, 0xf5, 0x3c, 0x4d, 0x96, 0xd3, 0xd9, 0xc2, 0x77, 0xf6, + 0xdc, 0x81, 0xc7, 0x35, 0xc9, 0x18, 0x78, 0x2f, 0xc4, 0xf5, 0xc2, 0x77, 0xf7, 0xdc, 0x41, 0x87, + 0xd3, 0x9a, 0xed, 0x43, 0xf3, 0x48, 0xca, 0x6c, 0xe1, 0x37, 0xf6, 0xdc, 0x41, 0xf7, 0xc9, 0xf6, + 0x81, 0xb6, 0x77, 0x80, 0x6c, 0xae, 0x84, 0x68, 0x93, 0xa7, 0x51, 0x16, 0xcf, 0x26, 0xbe, 0xb7, + 0xe7, 0x0c, 0x7a, 0x5c, 0x93, 0xac, 0x0f, 0xcd, 0x70, 0x36, 0x16, 0x57, 0x7e, 0x73, 0xcf, 0x19, + 0x74, 0xb8, 0x22, 0x90, 0xfb, 0x49, 0x2c, 0x92, 0xb1, 0xbf, 0xa9, 0xb8, 0x44, 0x04, 0x07, 0xd0, + 0xe1, 0xe9, 0xea, 0x65, 0x24, 0xb3, 0xf8, 0x8a, 0x7d, 0x0b, 0x3c, 0x9e, 0xae, 0xd4, 0x1d, 0xbb, + 0x4f, 0xb6, 0xcc, 0xb9, 0x3c, 0x5d, 0x71, 0x12, 0x05, 0x2f, 0xa1, 0x73, 0x1e, 0x4f, 0x66, 0x62, + 0x8c, 0x6e, 0xbd, 0x07, 0xee, 0xab, 0x14, 0xd5, 0x9d, 0x75, 0x75, 0x94, 0xa0, 0xc2, 0x99, 0x98, + 0xf8, 0x8d, 0x5a, 0x85, 0x33, 0x31, 0x09, 0x9e, 0xc1, 0x36, 0x4f, 0x57, 0xe1, 0x58, 0xcc, 0x64, + 0xfc, 0x9b, 0x58, 0x64, 0x04, 0x48, 0x71, 0x07, 0x4f, 0x1d, 0x5a, 0x80, 0xd4, 0x30, 0x20, 0x05, + 0x0f, 0x61, 0x33, 0x1c, 0xfe, 0x3c, 0x5e, 0x48, 0xb6, 0x03, 0x6e, 0x38, 0xd4, 0x1b, 0x70, 0x19, + 0x84, 0xf0, 0xce, 0xc9, 0x95, 0xcc, 0xa2, 0x91, 0x14, 0xe3, 0x70, 0xa8, 0xa0, 0x66, 0xdb, 0xd0, + 0x08, 0x87, 0x74, 0x57, 0x8f, 0x37, 0xc2, 0x21, 0xdb, 0x07, 0xef, 0x22, 0x4a, 0x34, 0xc8, 0x3b, + 0xe6, 0x72, 0xca, 0x2c, 0x27, 0x69, 0x70, 0x59, 0x32, 0x95, 0xe3, 0xf4, 0x2e, 0x6c, 0x12, 0x7a, + 0xea, 0xd0, 0x0e, 0xcf, 0x29, 0xf6, 0xd4, 0x84, 0x59, 0x59, 0xfd, 0xa6, 0xb1, 0xba, 0x76, 0xa1, + 0x22, 0x07, 0x82, 0xc7, 0xd0, 0x7a, 0x21, 0xae, 0xc9, 0x17, 0xed, 0xa9, 0x63, 0x79, 0xfa, 0x6f, + 0x07, 0xee, 0x17, 0xbb, 0x5f, 0x47, 0x97, 0x89, 0xb8, 0x88, 0x92, 0xa5, 0x60, 0xfb, 0xda, 0x6f, + 0xa7, 0xee, 0xfe, 0xa7, 0x1b, 0x84, 0x05, 0x7b, 0xbf, 0xc0, 0x0e, 0xd5, 0xde, 0x31, 0x6a, 0xf9, + 0x91, 0xa7, 0x1b, 0x79, 0xd6, 0x3d, 0x82, 0xf6, 0xf1, 0x79, 0x48, 0xa6, 0x7d, 0x77, 0xcf, 0x19, + 0xb8, 0xa7, 0x1b, 0xbc, 0xe0, 0xb0, 0x87, 0xd0, 0x7a, 0xb9, 0x94, 0xe2, 0x2a, 0x1c, 0x52, 0xb6, + 0x79, 0xa7, 0x1b, 0x5c, 0x33, 0x70, 0x27, 0x2d, 0x5f, 0x88, 0x6b, 0x95, 0x72, 0xb8, 0x53, 0x73, + 0x58, 0x1f, 0xbc, 0xe3, 0x34, 0x4d, 0x28, 0xed, 0xda, 0x78, 0x1a, 0x52, 0xc7, 0x2d, 0x68, 0x92, + 0xe1, 0xe0, 0xf7, 0xd0, 0x2f, 0x3b, 0x97, 0x87, 0x8b, 0x81, 0x8b, 0xf6, 0x9c, 0xdc, 0x1e, 0x12, + 0x6c, 0x87, 0x42, 0xd8, 0xc8, 0xcf, 0xc7, 0x20, 0x3e, 0x85, 0x4d, 0x32, 0xa3, 0x1e, 0x50, 0xf7, + 0xc9, 0xe3, 0x1a, 0xc0, 0x0d, 0x64, 0x3c, 0x57, 0x3e, 0xee, 0x10, 0xe2, 0x9f, 0x66, 0xe1, 0x30, + 0xf8, 0x71, 0x15, 0x5c, 0x8a, 0x25, 0x06, 0xe2, 0x2c, 0x9a, 0x0a, 0x75, 0x3e, 0xa7, 0x35, 0xf2, + 0x5e, 0x5f, 0xcf, 0x05, 0x5d, 0xa0, 0xc3, 0x69, 0x1d, 0xfc, 0xc1, 0x81, 0xed, 0xf2, 0x7e, 0xbc, + 0x93, 0x95, 0x1d, 0xb7, 0xdc, 0x89, 0xb4, 0x8a, 0xe4, 0x79, 0x56, 0x4d, 0x9e, 0xdd, 0x9b, 0xf6, + 0x55, 0xf3, 0xe7, 0x27, 0xe0, 0xbd, 0x8a, 0xe2, 0x6c, 0x2d, 0xc3, 0x77, 0x14, 0x84, 0x2e, 0x5d, + 0xd7, 0x55, 0xb1, 0x68, 0x3e, 0x4f, 0x97, 0x33, 0xa9, 0x30, 0xe4, 0x8a, 0x08, 0x4e, 0xa0, 0x83, + 0xfb, 0x95, 0xe3, 0x81, 0x32, 0x96, 0xa7, 0x95, 0x55, 0x7b, 0x90, 0xcb, 0xd5, 0x41, 0x45, 0x29, + 0x69, 0xd8, 0xa5, 0xe4, 0x14, 0x00, 0xa5, 0x0b, 0x65, 0x67, 0x1f, 0x9a, 0x44, 0xe5, 0x20, 0x54, + 0x0d, 0x29, 0xe1, 0x0d, 0x96, 0x1e, 0x63, 0x01, 0x93, 0x3f, 0xf8, 0x08, 0xc5, 0x2a, 0x21, 0xf1, + 0x36, 0x2e, 0xcf, 0x53, 0x66, 0x09, 0x6d, 0x05, 0x5d, 0xba, 0x32, 0x06, 0x1c, 0xcb, 0x00, 0x72, + 0xb1, 0xac, 0x0c, 0xb5, 0x9f, 0x44, 0xe0, 0xb3, 0xe5, 0xe9, 0xca, 0x40, 0x92, 0x53, 0xec, 0xdb, + 0xfa, 0x14, 0x8f, 0x7c, 0xbe, 0x67, 0x3d, 0x25, 0xbc, 0x85, 0x3e, 0xf6, 0xd7, 0x00, 0x3f, 0xcb, + 0xd2, 0xe5, 0x9c, 0x40, 0x63, 0x03, 0x68, 0x12, 0x95, 0xfb, 0xc7, 0xcc, 0x26, 0x7d, 0x37, 0xae, + 0x14, 0xea, 0x41, 0xc7, 0xe0, 0x1c, 0x4d, 0x26, 0xea, 0xa5, 0x71, 0x5c, 0x62, 0x2a, 0xb5, 0x2f, + 0xa2, 0xa4, 0x10, 0x5f, 0x44, 0x49, 0xee, 0x37, 0x2e, 0xcb, 0x66, 0x5c, 0x6d, 0xe6, 0x21, 0xb4, + 0x3f, 0x49, 0xd2, 0x48, 0xa2, 0x32, 0xda, 0x72, 0x78, 0x41, 0xb3, 0x43, 0x80, 0xa1, 0x18, 0xc5, + 0xd3, 0x28, 0x41, 0xa9, 0x57, 0x2d, 0x00, 0xb9, 0x8c, 0x5b, 0x4a, 0xc1, 0x53, 0x68, 0xe5, 0x54, + 0x3d, 0xf6, 0xc8, 0x3d, 0x1f, 0x45, 0x89, 0xd0, 0xb7, 0x20, 0x22, 0xf8, 0x0c, 0xb6, 0x54, 0x32, + 0x62, 0x6b, 0x3a, 0x17, 0xf2, 0x0e, 0xa9, 0x78, 0xa7, 0x26, 0x17, 0xfc, 0xd5, 0x01, 0x0f, 0x57, + 0xda, 0x80, 0x63, 0x0c, 0xd8, 0xaf, 0xd1, 0x53, 0xaf, 0x91, 0xed, 0x41, 0xf7, 0x5c, 0x62, 0x0f, + 0x34, 0x65, 0xac, 0xc3, 0x6d, 0x16, 0xe2, 0x15, 0xce, 0xa4, 0x09, 0xb7, 0xcb, 0x0b, 0x9a, 0x3d, + 0x82, 0x0e, 0xd6, 0x26, 0x25, 0xc4, 0x42, 0xd6, 0xe6, 0x86, 0xc1, 0x76, 0x01, 0x34, 0xb2, 0x4b, + 0x41, 0xd5, 0xcc, 0xe1, 0x16, 0x27, 0xf8, 0x10, 0x5a, 0x78, 0xd3, 0x97, 0xd1, 0xdc, 0xf8, 0xe6, + 0xdc, 0xe6, 0xdb, 0x5f, 0x1a, 0xd0, 0xfb, 0xc5, 0x52, 0x64, 0xd7, 0x5c, 0xfc, 0x6e, 0x29, 0x16, + 0x12, 0xb1, 0x25, 0x5a, 0xe7, 0x32, 0x11, 0x98, 0xb5, 0xe7, 0x9f, 0x47, 0xd9, 0x58, 0x21, 0xe5, + 0xf1, 0x9c, 0x42, 0x5f, 0x0d, 0xe6, 0x0b, 0xf2, 0xb5, 0xcd, 0x6d, 0x16, 0xe5, 0xbb, 0x98, 0xa6, + 0x52, 0x3b, 0x93, 0x53, 0x6c, 0x00, 0xf7, 0x4e, 0xae, 0x46, 0xc9, 0x72, 0x2c, 0x78, 0xba, 0x52, + 0xbb, 0xa9, 0x38, 0xf3, 0x2a, 0x9b, 0x7d, 0x07, 0x8b, 0x1b, 0xb1, 0x74, 0x69, 0x6a, 0x91, 0x62, + 0x85, 0xcb, 0x0e, 0xa1, 0x77, 0x32, 0xbd, 0x14, 0xe3, 0xb1, 0x18, 0x0f, 0x23, 0x19, 0xf9, 0xed, + 0xba, 0x01, 0xa2, 0xa4, 0xc2, 0xf6, 0x61, 0xeb, 0x55, 0x26, 0x5e, 0x67, 0xd1, 0x6c, 0x91, 0x44, + 0x52, 0x8c, 0xfd, 0x0e, 0x59, 0x2e, 0x33, 0x83, 0x2f, 0x1d, 0xd8, 0xca, 0x31, 0x5a, 0xcc, 0xd3, + 0xd9, 0x42, 0x60, 0x22, 0x9c, 0x64, 0x99, 0x4e, 0x84, 0x93, 0x2c, 0x63, 0x1f, 0x42, 0x8b, 0x8b, + 0xc5, 0x32, 0x91, 0x3a, 0x97, 0x1e, 0x98, 0x73, 0xf5, 0xde, 0x65, 0x22, 0xb9, 0xd6, 0x62, 0x3f, + 0x85, 0xed, 0x52, 0xb6, 0xea, 0xe6, 0xf1, 0x0d, 0xb3, 0xaf, 0x24, 0xe7, 0x15, 0xf5, 0xe0, 0x7f, + 0x4d, 0xe8, 0x5a, 0x96, 0x8b, 0x54, 0x44, 0x14, 0xb7, 0xf2, 0x54, 0x7c, 0x8f, 0x26, 0xbf, 0x1b, + 0x66, 0x23, 0xac, 0x5c, 0x3d, 0x70, 0xce, 0xf2, 0xe4, 0x75, 0xce, 0x4c, 0xb9, 0x74, 0x6f, 0x2b, + 0x97, 0x38, 0x47, 0x7e, 0x1e, 0xcd, 0x26, 0x62, 0x4c, 0xc9, 0xdb, 0xe6, 0x9a, 0x64, 0x07, 0xa6, + 0x76, 0x50, 0xb4, 0x4b, 0x15, 0x49, 0x4b, 0xb8, 0xa9, 0x2f, 0xaa, 0x16, 0xe2, 0xfc, 0xd0, 0x52, + 0x59, 0xa5, 0x28, 0xf6, 0x23, 0xd8, 0xfe, 0x34, 0x19, 0x9b, 0x3a, 0xb7, 0xc8, 0x63, 0xd9, 0x37, + 0xd6, 0x8c, 0x90, 0x57, 0x74, 0xd9, 0xc7, 0xd5, 0x71, 0x8e, 0xa2, 0xda, 0x7d, 0xe2, 0x97, 0xfc, + 0xb7, 0xe4, 0xbc, 0x3a, 0xfe, 0x1d, 0x5a, 0xf3, 0xa5, 0x0f, 0xb4, 0xf9, 0xbe, 0xd9, 0x5c, 0x88, + 0xb8, 0x35, 0x85, 0x7e, 0x64, 0xf7, 0x1d, 0xbf, 0x4b, 0x7b, 0xfa, 0x65, 0xfc, 0x94, 0x8c, 0xdb, + 0xfd, 0xe9, 0xd0, 0x6a, 0x7a, 0x7e, 0xaf, 0x7a, 0x50, 0x21, 0xe2, 0x56, 0x6b, 0x0c, 0x6b, 0x66, + 0x41, 0x7f, 0x8b, 0xb6, 0xd6, 0x0f, 0x7a, 0x4a, 0x85, 0xd7, 0x4c, 0x90, 0x1f, 0x57, 0xa7, 0x06, + 0x7f, 0xbb, 0x0a, 0x54, 0x59, 0xce, 0xab, 0x53, 0xc6, 0xa1, 0x35, 0xb8, 0xfb, 0xf7, 0xaa, 0xf7, + 0x2f, 0x44, 0xdc, 0x1a, 0xef, 0x7f, 0x08, 0x5d, 0x3b, 0xb0, 0x3b, 0xb4, 0xe9, 0x41, 0x5d, 0x60, + 0x17, 0xdc, 0xd6, 0x0c, 0xfe, 0xde, 0x80, 0xad, 0x70, 0x3a, 0x4f, 0x33, 0x69, 0x95, 0x2a, 0xf5, + 0x89, 0xe1, 0xd4, 0x7e, 0x62, 0x34, 0x2a, 0xcd, 0x98, 0x4a, 0x16, 0x95, 0x28, 0x8f, 0x2b, 0xc2, + 0x4a, 0x40, 0xaf, 0x94, 0x80, 0x8f, 0xa0, 0xa3, 0x5e, 0x1b, 0x8a, 0x9a, 0x24, 0x32, 0x0c, 0xf5, + 0xd1, 0xb3, 0xa2, 0x81, 0xb6, 0x45, 0x23, 0xb2, 0x26, 0xb1, 0x3c, 0x2b, 0x35, 0x12, 0xb6, 0x49, + 0x68, 0x71, 0x50, 0xfe, 0x3a, 0x9e, 0x8a, 0x85, 0x8c, 0xa6, 0x73, 0xac, 0x77, 0xee, 0xc0, 0xe5, + 0x16, 0x07, 0x4b, 0x1d, 0x39, 0xf1, 0x3c, 0x13, 0x58, 0x79, 0x8e, 0x24, 0xa5, 0xae, 0xcb, 0x2b, + 0x5c, 0xd4, 0x23, 0xb7, 0x8c, 0x1e, 0x28, 0xbd, 0x32, 0x97, 0xda, 0x75, 0x22, 0xa2, 0x8c, 0x12, + 0xb2, 0xcd, 0x15, 0x11, 0xfc, 0xab, 0x01, 0x4c, 0x21, 0xa9, 0x06, 0xd2, 0xaf, 0x0d, 0xce, 0xdb, + 0x61, 0x2b, 0x83, 0xd3, 0x5a, 0x03, 0xe7, 0xdd, 0x62, 0x8c, 0x56, 0xc0, 0xe4, 0x14, 0xf6, 0x18, + 0xd3, 0xe1, 0x14, 0xaa, 0x0e, 0xb7, 0x59, 0x2c, 0x80, 0x9e, 0xd5, 0x5e, 0xf1, 0xbd, 0xa3, 0xed, + 0x12, 0xaf, 0x06, 0x5a, 0xb8, 0x23, 0xb4, 0xdd, 0xdb, 0xa1, 0xed, 0xd9, 0xd0, 0x7e, 0xe9, 0x40, + 0xef, 0x48, 0xa6, 0xd3, 0x78, 0xc4, 0xc5, 0x28, 0xcd, 0xc6, 0x37, 0x83, 0xaa, 0xe0, 0x6b, 0xd8, + 0xf0, 0x1d, 0x80, 0x1b, 0x7e, 0x91, 0xe5, 0xc5, 0xf7, 0x91, 0x35, 0x00, 0xae, 0xc5, 0x8a, 0xa3, + 0x22, 0x7b, 0x1f, 0x1a, 0x61, 0x46, 0x99, 0x5b, 0x6a, 0x1b, 0xa5, 0x47, 0xc2, 0x1b, 0x61, 0x16, + 0x7c, 0x00, 0x7d, 0x75, 0x29, 0x2d, 0xca, 0xdb, 0x58, 0x1f, 0x9a, 0x27, 0x59, 0x96, 0xea, 0x46, + 0xa6, 0x88, 0xe0, 0x0a, 0xfa, 0x45, 0xf3, 0xc3, 0xc0, 0xbc, 0x4d, 0x7e, 0xd4, 0xfd, 0x51, 0xd8, + 0x83, 0xee, 0x59, 0x2a, 0x3f, 0xcb, 0x62, 0x49, 0xb5, 0x46, 0xf5, 0x0e, 0x9b, 0x15, 0x7c, 0x17, + 0x1e, 0x54, 0x4e, 0x36, 0xfd, 0x16, 0x53, 0xca, 0x35, 0x5f, 0xd7, 0xe7, 0x70, 0xbf, 0x50, 0x0d, + 0x87, 0x6f, 0x75, 0xc7, 0x75, 0xa3, 0xdf, 0xb3, 0x3c, 0x27, 0xa3, 0xf9, 0xf1, 0x35, 0xde, 0x04, + 0xc7, 0xe0, 0xe7, 0x68, 0xaa, 0x1f, 0x1e, 0xf9, 0x0d, 0x2e, 0x62, 0xb1, 0xba, 0xe9, 0xbb, 0x8d, + 0xa6, 0x92, 0x06, 0xfd, 0x26, 0xa1, 0x75, 0xf0, 0xc7, 0x06, 0xf4, 0xeb, 0x8c, 0x98, 0xe4, 0x72, + 0xac, 0xe4, 0x62, 0xcf, 0xa0, 0xf9, 0x45, 0x2c, 0x56, 0x7a, 0xc2, 0x08, 0xd6, 0x42, 0xbe, 0x76, + 0x13, 0xae, 0x36, 0xe0, 0xd3, 0x3a, 0x1a, 0xc9, 0x38, 0x9d, 0xe9, 0x8f, 0x0e, 0x45, 0xe1, 0x39, + 0xc7, 0x49, 0x3a, 0xfa, 0xad, 0xfa, 0x9c, 0xe6, 0x8a, 0xa8, 0x79, 0x2a, 0xcd, 0x3b, 0x3e, 0x95, + 0xcd, 0xda, 0xa7, 0x32, 0x80, 0x7b, 0xbf, 0x9c, 0x8f, 0x23, 0x29, 0x4e, 0xae, 0xe2, 0x85, 0x14, + 0xb3, 0x91, 0xc8, 0x27, 0xb8, 0x2a, 0x3b, 0xf8, 0x9b, 0xa3, 0x51, 0xb5, 0x46, 0xc8, 0xaf, 0x8c, + 0xad, 0x79, 0x4a, 0xae, 0x7e, 0x4a, 0xbe, 0x9a, 0x83, 0xcd, 0xb8, 0xaf, 0x49, 0x9c, 0xbd, 0x71, + 0x49, 0x7f, 0x5d, 0x3c, 0x8a, 0x67, 0x41, 0x7f, 0x45, 0xfd, 0x5a, 0x87, 0x65, 0xb3, 0x0e, 0x96, + 0xe0, 0x57, 0xa5, 0x0e, 0x87, 0x46, 0x8f, 0x26, 0x93, 0x4c, 0x4c, 0x22, 0xa9, 0x33, 0xc2, 0x30, + 0xd8, 0x07, 0xb0, 0x49, 0xca, 0x3a, 0xa8, 0xf5, 0x23, 0x4e, 0xae, 0x73, 0xbc, 0xf3, 0x8f, 0x37, + 0xbb, 0xce, 0x3f, 0xdf, 0xec, 0x3a, 0xff, 0x79, 0xb3, 0xeb, 0xfc, 0xe9, 0xbf, 0xbb, 0x1b, 0x97, + 0x9b, 0xf4, 0xb7, 0xef, 0xfb, 0xff, 0x0f, 0x00, 0x00, 0xff, 0xff, 0x8c, 0xa8, 0x3e, 0x99, 0xfd, + 0x13, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -5152,6 +5162,16 @@ func (m *ImportRoaringRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.UpdateExistence { + i-- + if m.UpdateExistence { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x38 + } if m.FieldCreatedAt != 0 { i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) i-- @@ -6400,6 +6420,9 @@ func (m *ImportRoaringRequest) Size() (n int) { if m.FieldCreatedAt != 0 { n += 1 + sovPublic(uint64(m.FieldCreatedAt)) } + if m.UpdateExistence { + n += 2 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -12610,6 +12633,26 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { break } } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdateExistence", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.UpdateExistence = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index 51de362bd..077ed87f9 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -251,6 +251,7 @@ message ImportRoaringRequest { uint64 Block = 4; int64 IndexCreatedAt = 5; int64 FieldCreatedAt = 6; + bool UpdateExistence = 7; } message ImportColumnAttrsRequest { diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 4abe89072..4773ef666 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -15,6 +15,9 @@ package pilosa import ( + "bytes" + "github.com/pilosa/pilosa/v2/roaring" + "reflect" "testing" ) @@ -47,10 +50,12 @@ type memAttrStore struct { store map[uint64]map[string]interface{} } -func (s *memAttrStore) Path() string { return "" } -func (s *memAttrStore) Open() error { return nil } -func (s *memAttrStore) Close() error { return nil } -func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return s.store[id], nil } +func (s *memAttrStore) Path() string { return "" } +func (s *memAttrStore) Open() error { return nil } +func (s *memAttrStore) Close() error { return nil } +func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { + return s.store[id], nil +} func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { s.store[id] = m return nil @@ -61,5 +66,33 @@ func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { } return nil } -func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil } -func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil } +func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil } +func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { + return nil, nil +} + +func TestAPI_CombineForExistence(t *testing.T) { + bm := roaring.NewBitmap() + bm.Add(pos(1, 1)) + bm.Add(pos(1, 2)) + bm.Add(pos(1, 65537)) //make sure to cross container boundary + bm.Add(pos(1, 65538)) + bm.Add(pos(2, 1)) + bm.Add(pos(2, 2)) + bm.Add(pos(2, 65537)) + bm.Add(pos(2, 65538)) + + buf := new(bytes.Buffer) + _, err := bm.WriteTo(buf) + panicOn(err) + raw := buf.Bytes() + results := combineForExistence(raw) + bm2 := roaring.NewBitmap() + bm2.ImportRoaringBits(results, false, false, 1< Date: Fri, 19 Feb 2021 18:08:25 -0600 Subject: [PATCH 03/11] Provide option to update existence on import roaring --- api.go | 23 +- cmd/sauron/design.org | 199 ---- cmd/sauron/httputil.go | 274 ----- cmd/sauron/main.go | 1802 --------------------------------- cmd/sauron/populate.sh | 10 - cmd/sauron/sample_schema.json | 1 - cmd/sauron/sauron_test.go | 396 -------- cmd/sauron/schema_test.go | 353 ------- cmd/sauron/sqlgen_test.go | 260 ----- cmd/sauron/test_sqlite.sh | 51 - cmd/sauron/vprint.go | 167 --- go.mod | 25 +- go.sum | 85 +- http/client_test.go | 6 + pilosa_internal_test.go | 35 +- 15 files changed, 109 insertions(+), 3578 deletions(-) delete mode 100644 cmd/sauron/design.org delete mode 100644 cmd/sauron/httputil.go delete mode 100644 cmd/sauron/main.go delete mode 100644 cmd/sauron/populate.sh delete mode 100644 cmd/sauron/sample_schema.json delete mode 100644 cmd/sauron/sauron_test.go delete mode 100644 cmd/sauron/schema_test.go delete mode 100644 cmd/sauron/sqlgen_test.go delete mode 100755 cmd/sauron/test_sqlite.sh delete mode 100644 cmd/sauron/vprint.go diff --git a/api.go b/api.go index 224c6bf15..f67ecf100 100644 --- a/api.go +++ b/api.go @@ -431,8 +431,15 @@ func importWorker(importWork chan importJob) { } if j.req.UpdateExistence { if ef := j.field.idx.existenceField(); ef != nil { - existence := combineForExistence(data) - ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false) + existence, err := combineForExistence(data) + if err != nil { + return errors.Wrap(err, "merging existence on roaring import") + } + + err = ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false) + if err != nil { + return errors.Wrap(err, "updating existence on roaring import") + } } } @@ -458,19 +465,21 @@ func importWorker(importWork chan importJob) { } // merge all rows to singled existence row -func combineForExistence(inputRoaringData []byte) []byte { +func combineForExistence(inputRoaringData []byte) ([]byte, error) { rowSize := uint64(1 << shardVsContainerExponent) rit, err := roaring.NewRoaringIterator(inputRoaringData) if err != nil { - panicOn(err) + return nil, err } bm := roaring.NewBitmap() - bm.MergeRoaringRawIteratorIntoExists(rit, rowSize) + err = bm.MergeRoaringRawIteratorIntoExists(rit, rowSize) + if err != nil { + return nil, err + } buf := new(bytes.Buffer) _, err = bm.WriteTo(buf) - panicOn(err) - return buf.Bytes() + return buf.Bytes(), err } // ImportRoaring is a low level interface for importing data to Pilosa when diff --git a/cmd/sauron/design.org b/cmd/sauron/design.org deleted file mode 100644 index aa579ab34..000000000 --- a/cmd/sauron/design.org +++ /dev/null @@ -1,199 +0,0 @@ - -* Oracle Design -- setup server that intersepts PQL queries and proxies to live pilosa server -- implement all the pilosa functions using SQL, specifically the sqlite flavor. This will - provide validation against pilosa results -- provide timings for each query path, PQL and SQL and compare -- this is intended to be a correctness check and not a performance check. the - sql solution is likely to be pretty slow with realitively small numbers ~1millions - #+BEGIN_SRC - .─────. - ,' `. ┌────────────┐ ┌────────┐ -;pilosa json: │ Sauron │ ┌──▶│ pilosa │ -: client ;──────────▶ │ │───┤ └────────┘ - ╲ ╱ │ │ │ ┌────────┐ - `. ,' └────────────┘ └──▶│ sqlite │ - `───' └────────┘ - #+END_SRC -* Setup Sql (sqlite) -#+name: setup test -#+header: :results silent -#+header: :db poc.db - #+BEGIN_SRC sqlite - create table bits (field string,row string, column bigint); - create table bsi (field string,column bigint , val bigint); - CREATE VIEW columns as - select distinct column from bits - UNION - select distinct column from bsi; - - INSERT INTO bits VALUES( 'color','red',1); - INSERT INTO bits VALUES( 'color','red',2); - INSERT INTO bits VALUES( 'color','red',3); - INSERT INTO bits VALUES( 'color','red',4); - INSERT INTO bits VALUES( 'color','red',5); - INSERT INTO bits VALUES( 'color','green',2); - INSERT INTO bits VALUES( 'color','green',4); - INSERT INTO bits VALUES( 'color','green',6); - INSERT INTO bits VALUES( 'color','green',8); - INSERT INTO bits VALUES( 'color','yellow',1); - INSERT INTO bits VALUES( 'color','yellow',3); - INSERT INTO bits VALUES( 'color','yellow',5); - INSERT INTO bits VALUES( 'color','yellow',7); - INSERT INTO bits VALUES( 'color','orange',5); - INSERT INTO bits VALUES( 'color','orange',6); - INSERT INTO bits VALUES( 'color','orange',7); - INSERT INTO bits VALUES( 'color','orange',8); - - INSERT INTO bsi VALUES( 'luminosity',1, 1); - INSERT INTO bsi VALUES( 'luminosity',2, 2); - INSERT INTO bsi VALUES( 'luminosity',3, 4); - INSERT INTO bsi VALUES( 'luminosity',4, 5); - INSERT INTO bsi VALUES( 'luminosity',5, 4); - INSERT INTO bsi VALUES( 'luminosity',6, 3); - INSERT INTO bsi VALUES( 'luminosity',7, 2); - INSERT INTO bsi VALUES( 'luminosity',8, 1); - #+END_SRC - -* TODO PQL to SQL TODO [10/11] -- [X] Row(color=red) - #+begin_src go -(*pql.Query)(&pql.Query{ - Calls: ([]*pql.Call)([]*pql.Call{ - (*pql.Call)(&pql.Call{ - Name: (string)("Row"), - Args: (map[string]interface{})(map[string]interface{}{ - (string)("color"): (string)("red"), - }), - Children: ([]*pql.Call)(nil), - Type: (pql.CallType)(0), - Precomputed: (map[uint64]interface{})(nil), - }), - }), - callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), - conditional: ([]string)(nil), -}) - #+end_src - #+BEGIN_SRC sqlite - select column from bits where field="color" AND row="red" - #+END_SRC -- [X] Count(Row(color=red)) - #+begin_src go -(*pql.Query)(&pql.Query{ - Calls: ([]*pql.Call)([]*pql.Call{ - (*pql.Call)(&pql.Call{ - Name: (string)("Count"), - Args: (map[string]interface{})(nil), - Children: ([]*pql.Call)([]*pql.Call{ - (*pql.Call)(&pql.Call{ - Name: (string)("Row"), - Args: (map[string]interface{})(map[string]interface{}{ - (string)("color"): (string)("red"), - }), - Children: ([]*pql.Call)(nil), - Type: (pql.CallType)(0), - Precomputed: (map[uint64]interface{})(nil), - }), - }), - Type: (pql.CallType)(0), - Precomputed: (map[uint64]interface{})(nil), - }), - }), - callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), - conditional: ([]string)(nil), -}) - - #+end_src - #+BEGIN_SRC sql - select count(*) from( - select column from bits where field="color" AND row="red" - ) - #+END_SRC -- [X] Intersect(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from bits where field="color" AND row="red" - intersect - select column from bits where field="color" AND row="green" - #+END_SRC -- [X] Union(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from bits where field="color" AND row="red" - union - select column from bits where field="color" AND row="green" - #+END_SRC -- [X] Difference(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from bits where field="color" AND row="red" - except - select column from bits where field="color" AND row="green" - #+END_SRC -- [X] Not(Row(color=red)) - #+BEGIN_SRC sql - select column from columns - except - select column from bits where field="color" AND row="red" - #+END_SRC -- [X] Xor(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from ( - select column from bits where field="color" AND row="green" - union - select column from bits where field="color" AND row="red" - ) - except - select column from ( - select column from bits where field="color" AND row="green" - intersect - select column from bits where field="color" AND row="red" - ) - #+END_SRC -- [X] Distinct(field=luminosity) - #+BEGIN_SRC sql - select distinct val from bsi where field="lumin" - #+END_SRC -- [X] Distinct(Row(color="red"),field=luminosity) - #+BEGIN_SRC sql - select distinct val from bsi - where field="luminosity" ANDcolumn in ( - select column from bits where row="color" AND row="red" - ) - #+END_SRC -- [X] Distinct(Intersect(Row(color="red"),Row(color="green")),luminosity) - #+BEGIN_SRC sql - select distinct val from bsi - where column in ( - select column from bits where field="color" AND row="red" - intersect - select column from bits where field="color" AND row="green" - ) - AND field="luminosity" - #+END_SRC - -- [X] Count(Union(Intersect(Row(color=red),Row(color=green)),Intersect(Row(color=yellow),Row(color=orange)))) -#+begin_src sqlite -select count(*) from ( -select column from ( - select column from bits where field="color" AND row="green" - intersect - select column from bits where field="color" AND row="red" -) -union -select column from ( - select column from bits where field="color" AND row="yellow" - intersect - select column from bits where field="color" AND row="orange" -)) - -- [ ] Rows(color) -#+begin_src sqlite -select distinct row from bits where field="color" -#+end_src - -- [ ] GroupBy(Rows(color)) - #+BEGIN_SRC sql - select row,count(*) from bits where field="color" group by row; - #+END_SRC -- [ ] GroupBy(Rows(color), Rows(other), limit=7) - #+BEGIN_SRC sql - select row,count(*) from bits where field="color" group by row; - #+END_SRC diff --git a/cmd/sauron/httputil.go b/cmd/sauron/httputil.go deleted file mode 100644 index 7056d40e8..000000000 --- a/cmd/sauron/httputil.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/json" - "log" - "mime" - "net/http" - "strings" - - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/pql" - "github.com/pkg/errors" -) - -// validHeaderAcceptJSON returns false if one or more Accept -// headers are present, but none of them are "application/json" -// (or any matching wildcard). Otherwise returns true. -func validHeaderAcceptJSON(header http.Header) bool { - return validHeaderAcceptType(header, "application", "json") -} - -func validHeaderAcceptType(header http.Header, typ, subtyp string) bool { - if v, found := header["Accept"]; found { - for _, v := range v { - t, _, err := mime.ParseMediaType(v) - if err != nil { - switch err { - case mime.ErrInvalidMediaParameter: - // This is an optional feature, so we can keep going anyway. - default: - continue - } - } - spl := strings.SplitN(t, "/", 2) - if len(spl) < 2 { - continue - } - switch { - case spl[0] == typ && spl[1] == subtyp: - return true - case spl[0] == "*" && spl[1] == subtyp: - return true - case spl[0] == typ && spl[1] == "*": - return true - case spl[0] == "*" && spl[1] == "*": - return true - } - } - return false - } - return true -} - -// successResponse is a general success/error struct for http responses. -type successResponse struct { - //h *Handler - Success bool `json:"success"` - Name string `json:"name,omitempty"` - CreatedAt int64 `json:"createdAt,omitempty"` - //Error *Error `json:"error,omitempty"` - Error error -} - -// check determines success or failure based on the error. -// It also returns the corresponding http status code. -func (r *successResponse) check(err error) (statusCode int) { - if err == nil { - r.Success = true - return 0 - } - - cause := errors.Cause(err) - - // Determine HTTP status code based on the error type. - switch cause.(type) { - case pilosa.BadRequestError: - statusCode = http.StatusBadRequest - case pilosa.ConflictError: - statusCode = http.StatusConflict - case pilosa.NotFoundError: - statusCode = http.StatusNotFound - default: - statusCode = http.StatusInternalServerError - } - - r.Success = false - r.Error = err // = &Error{Message: err.Error()} - - return statusCode -} - -// write sends a response to the http.ResponseWriter based on the success -// status and the error. -func (r *successResponse) write(w http.ResponseWriter, err error) { - // Apply the error and get the status code. - statusCode := r.check(err) - - // Marshal the json response. - msg, err := json.Marshal(r) - if err != nil { - http.Error(w, string(msg), http.StatusInternalServerError) - return - } - - // Write the response. - if statusCode == 0 { - w.Header().Set("Content-Type", "application/json") - _, err := w.Write(msg) - if err != nil { - log.Printf("error writing response: %v", err) - return - } - _, err = w.Write([]byte("\n")) - if err != nil { - log.Printf("error writing newline after response: %v", err) - return - } - } else { - http.Error(w, string(msg), statusCode) - } -} - -type postFieldRequest struct { - Options fieldOptions `json:"options"` -} - -// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values, -// and used for input validation. -type fieldOptions struct { - Type string `json:"type,omitempty"` - CacheType *string `json:"cacheType,omitempty"` - CacheSize *uint32 `json:"cacheSize,omitempty"` - Min *pql.Decimal `json:"min,omitempty"` - Max *pql.Decimal `json:"max,omitempty"` - Scale *int64 `json:"scale,omitempty"` - TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` - Keys *bool `json:"keys,omitempty"` - NoStandardView bool `json:"noStandardView,omitempty"` - ForeignIndex *string `json:"foreignIndex,omitempty"` -} - -func (o *fieldOptions) ToPilosaFieldOptions() *pilosa.FieldOptions { - - fo := &pilosa.FieldOptions{ - Type: o.Type, - - /* - //Base int64 `json:"base,omitempty"` - //BitDepth uint `json:"bitDepth,omitempty"` - Min: *o.Min, - Max: *o.Max, - Scale: *o.Scale, - NoStandardView: o.NoStandardView, - CacheSize: *o.CacheSize, - CacheType: *o.CacheType, - TimeQuantum: *o.TimeQuantum, - ForeignIndex: *o.ForeignIndex, - */ - } - if o.Keys != nil { - fo.Keys = *o.Keys - } - return fo -} - -func (o *fieldOptions) validate() error { - // Pointers to default values. - defaultCacheType := pilosa.DefaultCacheType - defaultCacheSize := uint32(pilosa.DefaultCacheSize) - - switch o.Type { - case pilosa.FieldTypeSet, "": - // Because FieldTypeSet is the default, its arguments are - // not required. Instead, the defaults are applied whenever - // a value does not exist. - if o.Type == "" { - o.Type = pilosa.FieldTypeSet - } - if o.CacheType == nil { - o.CacheType = &defaultCacheType - } - if o.CacheSize == nil { - o.CacheSize = &defaultCacheSize - } - if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type set")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type set")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) - } - case pilosa.FieldTypeInt: - if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) - } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { - return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) - } - case pilosa.FieldTypeDecimal: - if o.Scale == nil { - return pilosa.NewBadRequestError(errors.New("decimal field requires a scale argument")) - } else if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) - } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { - return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) - } - case pilosa.FieldTypeTime: - if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time")) - } else if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type time")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type time")) - } else if o.TimeQuantum == nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time")) - } - case pilosa.FieldTypeMutex: - if o.CacheType == nil { - o.CacheType = &defaultCacheType - } - if o.CacheSize == nil { - o.CacheSize = &defaultCacheSize - } - if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type mutex")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) - } - case pilosa.FieldTypeBool: - if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) - } else if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) - } else if o.Keys != nil { - return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool")) - } else if o.ForeignIndex != nil { - return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key")) - } - default: - return errors.Errorf("invalid field type: %s", o.Type) - } - return nil -} diff --git a/cmd/sauron/main.go b/cmd/sauron/main.go deleted file mode 100644 index 497e5346b..000000000 --- a/cmd/sauron/main.go +++ /dev/null @@ -1,1802 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - //"io" - "bytes" - "context" - "database/sql" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "log" - "net" - "net/http" - "os" - "os/exec" - "sort" - "strconv" - "strings" - "time" - - "modernc.org/sqlite" - _ "modernc.org/sqlite" - - //"github.com/gorilla/handlers" - "github.com/gorilla/mux" - "github.com/pilosa/pilosa/v2" - - "github.com/pilosa/pilosa/v2/encoding/proto" - "github.com/pilosa/pilosa/v2/pql" - "github.com/pilosa/pilosa/v2/roaring" - "github.com/pkg/errors" - "github.com/shurcooL/go-goon" -) - -func main() { - -} - -// Sauron sees everything. -// And he converts PQL into SQL. -// -// The PQL is applied to a Pilosa and the SQL -// is applied to a SQL database. Then the -// query results are compared as they are -// returned to the client. -// -type Sauron struct { - cfg *SauronConfig - - sql *PQLToSQL - url string // "https://..." - addr string // host:port - mux *http.ServeMux - srv *http.Server - cmd *exec.Cmd - - // target of proxy - pilosaHttpURL string - - Handler http.Handler -} - -type SauronConfig struct { - Bind string // host:port - BindGRPC string // host:port - GossipPort string // port - DataDir string - - DatabaseName string // for sql database -} - -// IndexInfo2 and FieldInfo2 let us look up -// the schema details quickly by using maps -// to index the fields in an index. -type IndexInfo2 struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options pilosa.IndexOptions `json:"options"` - Fields map[string]*FieldInfo2 `json:"fields"` - ShardWidth uint64 `json:"shardWidth"` -} - -type FieldInfo2 struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options pilosa.FieldOptions `json:"options"` - Views []*pilosa.ViewInfo `json:"views,omitempty"` - indexInfo *IndexInfo2 -} - -func (fi *FieldInfo2) GetIndexName() string { - return fi.indexInfo.Name -} - -func (fi *FieldInfo2) GetIndexCreatedAt() int64 { - return fi.indexInfo.CreatedAt -} - -func (fi *FieldInfo2) GetCreatedAt() int64 { - return fi.CreatedAt -} - -func (fi *FieldInfo2) GetName() string { - return fi.Name -} - -type PQLToSQL struct { - schema pilosa.Schema - DB *sql.DB - - dbprefix string // prefixed to all tables - - // map from indexName to *IndexInfo2 - i2f map[string]*IndexInfo2 - ShardWidth uint64 -} - -func NewPQLToSQL(dbprefix string) *PQLToSQL { - return &PQLToSQL{ - dbprefix: strings.ToLower(dbprefix), // some SQL are not case sensitive, just use lower case. - i2f: make(map[string]*IndexInfo2), - ShardWidth: 1048576, - } -} - -func (ps *PQLToSQL) Start() error { - db, err := sql.Open("sqlite", fmt.Sprintf("%v.db", ps.dbprefix)) - if err != nil { - log.Fatal(err) - } - ps.DB = db - return nil -} -func (ps *PQLToSQL) MustRemove() { - name := fmt.Sprintf("%v.db", ps.dbprefix) - os.Remove(name) - // panicOn(err) -} - -func (ps *PQLToSQL) Stop() error { - return ps.DB.Close() -} -func (ps *PQLToSQL) Store(index, field string, rowIdOrKey interface{}, rowCall []*pql.Call) (bool, error) { - rowSql, _, _, err := ToSql(rowCall, index) - panicOn(err) - insertSql := fmt.Sprintf(`insert into %vλbits select "%v" as field, "%v" as row, column,0 as timestamp from ( %v )`, index, field, rowIdOrKey, rowSql) - res, err := ps.DB.Exec(insertSql) - if err != nil { - return false, err - } - aff, err := res.RowsAffected() - if err != nil { - return false, err - } - return aff > 0, nil -} - -// convert '-' to 'Θ' since db cannot have '-' in table names, -// but pilosa allows it in index and field names. -func dash2theta(s string) (r string) { - return strings.Replace(s, "-", "Θ", -1) -} - -func theta2dash(s string) (r string) { - return strings.Replace(s, "Θ", "-", -1) -} -func (ps *PQLToSQL) CreateSchema(jsonBytes []byte) (err error) { - err = json.Unmarshal(jsonBytes, &ps.schema) - if err != nil { - panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) - } - for _, i := range ps.schema.Indexes { - ps.createDatabase(i.Name) - for _, f := range i.Fields { - panicOn(ps.CreateField(i, f)) - } - } - return nil -} - -func (ps *PQLToSQL) CreateFieldWithOptions(indexName, fieldName string, o fieldOptions) error { - - ii2, ok := ps.i2f[indexName] - if !ok { - return fmt.Errorf("index '%v' not found", indexName) - } - - f2, ok := ii2.Fields[fieldName] - if !ok { - f2 = &FieldInfo2{ - Name: fieldName, - //CreatedAt: f.CreatedAt, - Options: *(o.ToPilosaFieldOptions()), - //Views: f.Views, - indexInfo: ii2, - } - ii2.Fields[fieldName] = f2 - } - return nil -} - -func (ps *PQLToSQL) CreateIndex(indexName string, opts *pilosa.IndexOptions) (err error) { - ii2, ok := ps.i2f[indexName] - if !ok { - ii2 = &IndexInfo2{ - Name: indexName, - //CreatedAt: i.CreatedAt, - Options: *opts, - Fields: make(map[string]*FieldInfo2), - ShardWidth: 1048576, - } - //TODO (twg) figure oiut where to get Shardwidht - ps.i2f[indexName] = ii2 - } - return nil -} -func (ps *PQLToSQL) Query(index string, query []*pql.Call) (err error) { - // sql, err := ToSQL(query, index) - //TODO (twg) implement QUery - return nil -} -func (ps *PQLToSQL) CreateField(i *pilosa.IndexInfo, f *pilosa.FieldInfo) (err error) { - ii2, ok := ps.i2f[i.Name] - if !ok { - ii2 = &IndexInfo2{ - Name: i.Name, - CreatedAt: i.CreatedAt, - Options: i.Options, - Fields: make(map[string]*FieldInfo2), - ShardWidth: i.ShardWidth, - } - ps.i2f[i.Name] = ii2 - } - - f2, ok := ii2.Fields[f.Name] - if !ok { - f2 = &FieldInfo2{ - Name: f.Name, - CreatedAt: f.CreatedAt, - Options: f.Options, - Views: f.Views, - indexInfo: ii2, - } - ii2.Fields[f.Name] = f2 - } - return nil -} - -func (ps *PQLToSQL) createDatabase(index string) (err error) { - idx_prefix := dash2theta(index) + "λ" - // Create fields that don't exist. - sql := fmt.Sprintf("create table %vbits (field string,row,column,timestamp, unique(field,row,column,timestamp) )", idx_prefix) - _, err = ps.DB.Exec(sql) - panicOn(err) - sql = fmt.Sprintf("create table %vbsi (field,column, val bigint,unique(field,column)) ", idx_prefix) - _, err = ps.DB.Exec(sql) - panicOn(err) - sql = fmt.Sprintf(`CREATE VIEW %vcolumns as - select distinct column from %vbits - UNION - select distinct column from %vbsi`, idx_prefix, idx_prefix, idx_prefix) - _, err = ps.DB.Exec(sql) - panicOn(err) - return -} -func (ps *PQLToSQL) GetField(indexName, fieldName string) (*FieldInfo2, error) { - idx, ok := ps.i2f[indexName] - if !ok { - return nil, errors.New(fmt.Sprintf("index not found '%v'", indexName)) - } - f, ok := idx.Fields[fieldName] - if !ok { - return nil, errors.New(fmt.Sprintf("field not found '%v' in index '%v'", fieldName, indexName)) - } - return f, nil -} - -//TODO (twg) below will be moved to its own file later just have it here for quick navigation -type PqlFunction uint8 - -//debate between this -type ResultType uint8 - -const ( - UNKNOWN = ResultType(iota) - BITMAP - INT - ARRAY - GROUPCOUNT - SIGNEDROW - BOOL - PAIRSFIELD - TABLE - AGGSQL -) - -func getAggregateSql(call *pql.Call, index string) (string, error) { - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", errors.New("no _field present") - } - } - var sq string - if len(call.Children) > 0 { - //only child should be a bitmap call - var err error - sq, _, _, err = ToSql(call.Children, index) - if err != nil { - return sq, err - } - sq = " and column in (" + sq + ")" - - } - sqb := fmt.Sprintf(`select %v(val) from %vλbsi where field="%v"%v`, strings.ToLower(call.Name), index, fieldName, sq) - sql := fmt.Sprintf(`select val,count(*) from %vλbsi where val=(%v)%v`, index, sqb, sq) - - return sql, nil -} - -// ToSql takes a pilosa query(ast) and converts it to sql -// assumption: the pql has successfully executed on server -func applyCondition(field string, cond *pql.Condition) (sql string) { - - switch cond.Op { - case pql.EQ, pql.NEQ, pql.LT, pql.LTE, pql.GT, pql.GTE: - val, ok := cond.Uint64Value() - if !ok { - panic("no value for condition") - } - if cond.Op == pql.EQ { - sql = fmt.Sprintf(`field="%v" and val=%d`, field, val) - } else if cond.Op == pql.NEQ { - sql = fmt.Sprintf(`field="%v" and val!=%d`, field, val) - } else if cond.Op == pql.LT { - sql = fmt.Sprintf(`field="%v" and val<%d`, field, val) - } else if cond.Op == pql.LTE { - sql = fmt.Sprintf(`field="%v" and val<=%d`, field, val) - } else if cond.Op == pql.GT { - sql = fmt.Sprintf(`field="%v" and val>%d`, field, val) - } else if cond.Op == pql.GTE { - sql = fmt.Sprintf(`field="%v" and val>=%d`, field, val) - } - case pql.BETWEEN, pql.BTWN_LT_LTE, pql.BTWN_LTE_LT, pql.BTWN_LT_LT: - val, ok := cond.Uint64SliceValue() - if !ok { - panic("bad value for condition") - } - if cond.Op == pql.BETWEEN { - sql = fmt.Sprintf(`field="%v" and %v>=%d and %v<=%d`, field, "val", val[0], "val", val[1]) - } else if cond.Op == pql.BTWN_LT_LTE { - sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[1]) - } else if cond.Op == pql.BTWN_LTE_LT { - sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[0]) - } else if cond.Op == pql.BTWN_LT_LT { - sql = fmt.Sprintf(`field="%v" and %v>%d and %v<%d`, field, "val", val[0], "val", val[1]) - } - } - return -} -func FetchFields(rowsCalls []*pql.Call) ([]string, []interface{}, error) { - //TODO (twg) date fields in rows? - results := make([]string, len(rowsCalls)) - columns := make([]interface{}, 0) - for i, call := range rowsCalls { - //TODO (twg) Time Fields ie from/to args - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return nil, nil, errors.New("no field for rows") - } - } - results[i] = fieldName.(string) - col, ok := call.Args["column"] - if ok { - s := fmt.Sprintf("f%v.column='%v'", i+1, col) - columns = append(columns, s) - } - } - return results, columns, nil -} - -func ToSql(ast []*pql.Call, index string) (string, ResultType, []interface{}, error) { - resultType := UNKNOWN - optional := make([]interface{}, 0) - if len(ast) == 0 { - return "", resultType, optional, nil - } - stack := make([]string, 0) - for _, call := range ast { - switch c := strings.ToLower(call.Name); c { - case "row": - var sql string - if call.HasConditionArg() { - for k, v := range call.Args { - csql := applyCondition(k, v.(*pql.Condition)) - sql = fmt.Sprintf("select distinct column from %vλbsi where %v", index, csql) - - break - } - } else { - field, err := call.FieldArg() - panicOn(err) - row := call.Args[field] - - t, ok := call.Args["from"] - var timestampClause string - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause = fmt.Sprintf(` AND timestamp >= "%v"`, clean) - } - t, ok = call.Args["to"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause += fmt.Sprintf(` AND timestamp < "%v"`, clean) - - } - - sql = fmt.Sprintf(`select distinct column from %vλbits where field="%v" AND row="%v"%v`, index, field, row, timestampClause) - - } - stack = append(stack, sql) - resultType = BITMAP - case "count": - //should have args and should be top level - if len(call.Children) == 0 { - return "", resultType, optional, errors.New("invalid args for count") - } - nestedSQL, _, _, err := ToSql(call.Children, index) - panicOn(err) - sql := "select count(*) from( " + nestedSQL + " )" - stack = append(stack, sql) - resultType = INT - case "union", "intersect": - var sql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(sql) == 0 { - sql = sq - } else { - sql += " " + c + " " + sq - } - } - sql = "select column from(" + sql + ")" - stack = append(stack, sql) - resultType = BITMAP - case "difference": - var sql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(sql) == 0 { - sql = sq - } else { - sql += " except " + sq - } - } - sql = "select column from(" + sql + ")" - stack = append(stack, sql) - resultType = BITMAP - case "not": - //should have args and should be top level - if len(call.Children) == 0 { - return "", resultType, optional, errors.New("invalid args for count") - } - nestedSQL, _, _, err := ToSql(call.Children, index) - panicOn(err) - sql := fmt.Sprintf("select column from %vλcolumns except %v", index, nestedSQL) - stack = append(stack, sql) - resultType = BITMAP - case "xor": - var unionSql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(unionSql) == 0 { - unionSql = sq - } else { - unionSql += " union " + sq - } - } - unionSql = "select column from(" + unionSql + ")" - var intersectSql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(intersectSql) == 0 { - intersectSql = sq - } else { - intersectSql += " intersect " + sq - } - } - intersectSql = "select column from(" + intersectSql + ")" - sql := unionSql + " except " + intersectSql - stack = append(stack, sql) - resultType = BITMAP - - case "distinct": - //TODO validate field type, only valid for bsi fields - fieldName, ok := call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("field not specified on distinct call") - } - var sql string - if len(call.Children) > 1 { - return "", resultType, optional, errors.New(fmt.Sprintf("unexpected args to distinct query:'%v'", call.Children)) - } - sql = fmt.Sprintf(`select distinct val from %vλbsi where field="%v"`, index, fieldName) - if len(call.Children) > 0 { - sq, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) - panicOn(err) - sql += " AND column in (" + sq + ")" - } - stack = append(stack, sql) - resultType = SIGNEDROW - case "rows": - //TODO (twg) Time Fields ie from/to args - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("no _field present") - } - } - optional = append(optional, fieldName) - sql := fmt.Sprintf(`select distinct field, row from %vλbits where field="%v"`, index, fieldName) - col, ok := call.Args["column"] - if ok { - sql = fmt.Sprintf("%v and column='%v'", sql, col) - } - prev, ok := call.Args["previous"] - if ok { - sql = fmt.Sprintf("%v and row>'%v'", sql, prev) - } - t, ok := call.Args["from"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) - sql += timestampClause - } - t, ok = call.Args["to"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) - sql += timestampClause - } - sql += " order by row" - limit, ok := call.Args["limit"] - if ok { - sql = fmt.Sprintf("%v limit %v", sql, limit) - } - stack = append(stack, sql) - resultType = ARRAY - case "groupby": - //need to figure out how many fields we will be looking at - fields, optColumns, err := FetchFields(call.Children) - panicOn(err) - sql := "select " - for i := range fields { - f := fmt.Sprintf("f%d", i+1) - if i == 0 { - sql += fmt.Sprintf(" %s.field, %s.row", f, f) - } else { - - sql += fmt.Sprintf(",%s.field, %s.row", f, f) - } - } - tname := fmt.Sprintf("%vλbits", index) - sql += fmt.Sprintf(",count(*) as cnt from %v f1", tname) - if len(fields) > 1 { - for i := range fields[1:] { - f := fmt.Sprintf("f%d", i+2) - sql += fmt.Sprintf(" inner join %v %v on f1.column = %v.column", tname, f, f) - } - } - sql += " where " - gb := "" - for i, field := range fields { - f := fmt.Sprintf("f%d", i+1) - if i == 0 { - sql += fmt.Sprintf("%v.field='%v'", f, field) - gb += fmt.Sprintf("%v.field,%v.row", f, f) - } else { - sql += fmt.Sprintf("and %v.field='%v'", f, field) - gb += fmt.Sprintf(",%v.field,%v.row", f, f) - } - } - if len(optColumns) > 0 { - for _, column := range optColumns { - sql += fmt.Sprintf(" and %v ", column) - } - } - sql += " group by " + gb + " order by cnt desc" - - optional = append(optional, len(fields)) - - stack = append(stack, sql) - resultType = GROUPCOUNT - case "topn": - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("no _field present") - } - } - var sq string - if len(call.Children) > 0 { - if len(call.Children) > 1 { - return "", resultType, optional, errors.New("invalid topn") - } - var err error - sq, _, _, err = ToSql(call.Children, index) - panicOn(err) - sq = " and column in (" + sq + ")" - - } - sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) - - limit, ok := call.Args["n"] - if ok { - sql += fmt.Sprintf(" limit %v", limit) - } - stack = append(stack, sql) - resultType = PAIRSFIELD - case "topk": - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("no _field present") - } - } - var sq string - filter, ok := call.Args["filter"] - if ok { - //assuming this works in pilosa so the filter must be a correct bitmap call - //ie row/union/intersect/xor/difference - var err error - sq, _, _, err = ToSql([]*pql.Call{filter.(*pql.Call)}, index) - panicOn(err) - sq = " and column in (" + sq + ")" - - } - t, ok := call.Args["from"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) - sq += timestampClause - } - t, ok = call.Args["to"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) - sq += timestampClause - } - sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) - - limit, ok := call.Args["k"] - if ok { - sql += fmt.Sprintf(" limit %v", limit) - } - stack = append(stack, sql) - resultType = PAIRSFIELD - case "min", "max": - sql, err := getAggregateSql(call, index) - panicOn(err) - stack = append(stack, sql) - resultType = AGGSQL - case "sum": - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - panic("no _field present") - } - } - var sq string - if len(call.Children) > 0 { - //only child should be a bitmap call - var err error - sq, _, _, err = ToSql(call.Children, index) - panicOn(err) - sq = " and column in (" + sq + ")" - - } - sql := fmt.Sprintf(`select sum(val),count(*) from %vλbsi where field="%v"%v`, index, fieldName, sq) - stack = append(stack, sql) - resultType = AGGSQL - case "constrow": - sql := "select column1 as column from (values " - columns, ok := call.Args["columns"].([]interface{}) - - if ok { - for i, col := range columns { - if i != 0 { - sql += "," - } - sql += fmt.Sprintf("(%v)", col) - - } - sql += ")" - - } - - stack = append(stack, sql) - resultType = BITMAP - case "all": - sql := fmt.Sprintf("select column from %vλcolumns", index) - stack = append(stack, sql) - resultType = BITMAP - case "extract": - // extract is a table response - bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) - panicOn(err) - var rowsPart string - if len(call.Children) > 1 { - rows := call.Children[1:] - rowsPart = "select field,row from(" - for i, row := range rows { - part, _, _, err1 := ToSql([]*pql.Call{row}, index) - panicOn(err1) - if i > 0 { - rowsPart += "union all select field,row from (" - } - rowsPart += part + ")" - } - } - var sql string - if len(rowsPart) == 0 { - sql = "select '' as field, '' as row,column from (" + bm + ")" - } else { - sql = "select sq.field, sq.row, b.column from (" - sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) - sql += " where b.column in (" + bm + ")" - } - - stack = append(stack, sql) - resultType = TABLE - case "unionrows": - var rowsPart string - rowsPart = "select field,row from(" - for i, row := range call.Children { - part, _, _, err1 := ToSql([]*pql.Call{row}, index) - panicOn(err1) - if i > 0 { - rowsPart += "union all select field,row from (" - } - rowsPart += part + ")" - } - sql := "select distinct b.column from (" - sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) - stack = append(stack, sql) - resultType = BITMAP - case "includescolumn": - bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) - panicOn(err) - col, ok := call.Args["column"] - if !ok { - return "", resultType, optional, errors.New("missing column parameter") - } - - sql := fmt.Sprintf("select count(*)>0 from (select column from(%v) where column='%v')", bm, col) - stack = append(stack, sql) - resultType = BOOL - default: - goon.Dump(call) - return "", resultType, optional, errors.New(fmt.Sprintf("unknown call '%v'", c)) - } - } - if len(stack) == 0 { - //not sure if i should error on an NOP - return "", resultType, optional, nil - } - return stack[0], resultType, optional, nil -} - -func NewSauron(cfg *SauronConfig) *Sauron { - - addr := "127.0.0.1:8080" - url := "http://" + addr - s := &Sauron{ - cfg: cfg, - addr: addr, - url: url, - sql: NewPQLToSQL(cfg.DatabaseName), - srv: &http.Server{ - Addr: addr, - ReadTimeout: 50 * time.Millisecond, - WriteTimeout: 10 * time.Second, - MaxHeaderBytes: 1 << 20, - }, - } - s.Handler = newRouter(s) - s.srv.Handler = s.Handler - return s -} - -func (p *Sauron) Start() (urlstring string, err error) { - started := make(chan struct{}) - - panicOn(p.sql.Start()) - - // start a pilosa server - bind := p.cfg.Bind - dataDir := p.cfg.DataDir - bindGRPC := p.cfg.BindGRPC - gossipPort := p.cfg.GossipPort - - p.cmd = exec.Command("pilosa", "server", - "--bind", bind, - "--data-dir", dataDir, - "--bind-grpc", bindGRPC, - "--gossip.port", gossipPort) - err = p.cmd.Start() - panicOn(err) - - // wait for it to be listening - p.pilosaHttpURL = "http://" + bind - ok := false - seconds := 20 - for i := 0; i < seconds; i++ { - nc, err := net.Dial("tcp", bind) - if err == nil { - nc.Close() - ok = true - break - } - time.Sleep(time.Second) - } - if !ok { - return "", fmt.Errorf("could not contact pilosa server on %v after %v seconds. pid = %v", bind, seconds, p.cmd.Process.Pid) - } - go func() { - close(started) - - err = p.srv.ListenAndServe() - if err != nil { - if strings.Contains(err.Error(), "Server closed") { - return - } - } - panicOn(err) - }() - <-started - return p.url, nil -} - -func (p *Sauron) Stop() { - panicOn(p.sql.Stop()) - err := p.srv.Shutdown(context.Background()) - panicOn(err) - // Kill pilosa - if err = p.cmd.Process.Kill(); err != nil { - vv("failed to kill process: ", err) - } -} - -// GetAvailPort asks the OS for an unused port. -// There's a race here, where the port could be grabbed by someone else -// before the caller gets to Listen on it, but we are only using -// it to find a random port for the test hang debugging. -// Moreover, in practice such races are rare. Just ask for -// it again if the port is taken. -// Uses net.Listen("tcp", ":0") to determine a free port, then -// releases it back to the OS with Listener.Close(). -func GetAvailPort() int { - l, _ := net.Listen("tcp", ":0") - r := l.Addr() - l.Close() - return r.(*net.TCPAddr).Port -} - -// newRouter creates a new mux http router. -func newRouter(handler *Sauron) http.Handler { - router := mux.NewRouter() - - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") - router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") - router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") - - router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex") - router.HandleFunc("/index/{index}/field/", handler.handlePostField).Methods("POST").Name("PostField") - router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField") - router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring") - router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport") - - return router -} - -// handleGetSchema handles GET /schema requests. -func (s *Sauron) handleGetSchema(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - _ = body - _ = resp - _ = err - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(resp.StatusCode) - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - _, err = w.Write(respBody) - panicOn(err) - - // B) there is no SQL version of get schema. - -} - -// creates an entire schema from the dumped schema. -// requires completed structures, no defaults applied -func (s *Sauron) handlePostSchema(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - - w.WriteHeader(resp.StatusCode) - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - _, err = w.Write(respBody) - panicOn(err) - - // B) SQL - panicOn(s.sql.CreateSchema(body)) -} - -// handlePostField handles /index/{index}/field/{field} requests -func (s *Sauron) handlePostIndex(w http.ResponseWriter, r *http.Request) { - body, resp, err := s.ForwardHandler(w, r) - _ = body - _ = resp - _ = err - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - indexName, ok := mux.Vars(r)["index"] - if !ok { - http.Error(w, "index name is required", http.StatusBadRequest) - return - } - req := struct { - Options pilosa.IndexOptions - }{ - Options: pilosa.IndexOptions{ - Keys: false, - TrackExistence: true, - }} - if len(body) > 0 { //only process if options are present - err = json.Unmarshal(body, &req) - panicOn(err) - } - err = s.sql.CreateIndex(indexName, &req.Options) - panicOn(err) - w.Write(respBody) -} - -// handlePostField handles /index/{index}/field/{field} requests -func (s *Sauron) handlePostField(w http.ResponseWriter, r *http.Request) { - body, resp, err := s.ForwardHandler(w, r) - _ = body - _ = resp - _ = err - indexName, ok := mux.Vars(r)["index"] - if !ok { - http.Error(w, "index name is required", http.StatusBadRequest) - return - } - - fieldName, ok := mux.Vars(r)["field"] - if !ok { - http.Error(w, "field name is required", http.StatusBadRequest) - return - } - - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - // Decode request. - var req postFieldRequest - dec := json.NewDecoder(bytes.NewReader(body)) - dec.DisallowUnknownFields() - err = dec.Decode(&req) - if err != io.EOF { - panicOn(err) - } - err = s.sql.CreateFieldWithOptions(indexName, fieldName, req.Options) - _, err = w.Write(respBody) - panicOn(err) -} - -// handlePostQuery handles /index/{index}/query requests -func (s *Sauron) handlePostQuery(w http.ResponseWriter, req *http.Request) { - - body, resp, err := s.ForwardHandler(w, req) - pilosaJSON, err := ioutil.ReadAll(resp.Body) - panicOn(err) - var pilosaResponse pilosa.QueryResponse // DEBUG - dec := json.NewDecoder(bytes.NewReader(pilosaJSON)) - err = dec.Decode(&pilosaResponse) - panicOn(err) - qreq := &pilosa.QueryRequest{ - Index: mux.Vars(req)["index"], - Query: string(body), - } - - oracleResponse, err := s.sql.IssueQuery(qreq) - - if !pilosaResponse.Equals(oracleResponse) { - //plan is to write out a useful query response indicating a discrepancy - //TODO (twg) write diff in response somehow - vv("oracle:") - prettyJson(*oracleResponse) - vv("pilosa:") - prettyJson(pilosaResponse) - panic("barf") - } -} -func prettyJson(a pilosa.QueryResponse) { - src, _ := a.MarshalJSON() - dst := &bytes.Buffer{} - if err := json.Indent(dst, src, "", " "); err != nil { - panic(err) - } - vv("%v", string(dst.Bytes())) -} - -func (s *Sauron) FieldInfo(index, field string) (*FieldInfo2, error) { - idx, ok := s.sql.i2f[index] - if ok { - f, ok := idx.Fields[field] - if ok { - return f, nil - } - return nil, errors.New(fmt.Sprintf("field not in schema '%v'", field)) - } - return nil, errors.New(fmt.Sprintf("index not in schema '%v'", index)) -} -func (s *Sauron) handlePostImport(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - _ = body - _ = resp - _ = err - // Get index and field type to determine how to handle the - // import data. - indexName := mux.Vars(req)["index"] - fieldName := mux.Vars(req)["field"] - - _ = indexName - _ = fieldName - - // If the clear flag is true, treat the import as clear bits. - q := req.URL.Query() - doClear := q.Get("clear") == "true" - doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" - _ = doClear - _ = doIgnoreKeyCheck - - info, err := s.FieldInfo(indexName, fieldName) - panicOn(err) - // Unmarshal request based on field type. - fieldType := info.Options.Type - if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal { - iv := &pilosa.ImportValueRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, iv); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - //TODO (twg) ignore columnKeys at the moment - //do importValue - for i, column := range iv.ColumnIDs { - if len(iv.Values) > 0 { - s.sql.Set(indexName, column, fieldName, iv.Values[i], "") - } else if len(iv.FloatValues) > 0 { //TODO (twg) these may not be reachable - s.sql.Set(indexName, column, fieldName, iv.FloatValues[i], "") //probably some issues here - } else if len(iv.StringValues) > 0 { - s.sql.Set(indexName, column, fieldName, iv.StringValues[i], "") - } - } - } else { - // Field type: set, time, mutex - // Marshal into request object. - ir := &pilosa.ImportRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, ir); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - //do import - action := s.sql.Set - if ir.Clear { - action = func(index string, column uint64, field string, value interface{}, ts string) (bool, error) { - return s.sql.Clear(index, column, field, value) - } - } - changed := false - _ = changed - for i, column := range ir.ColumnIDs { - ts := "" - if len(ir.Timestamps) > 0 { - t := time.Unix(ir.Timestamps[i], 0) - ts = t.Format("2006-01-02 15:04:05") - } - if len(ir.RowIDs) > 0 { - change, err := action(indexName, column, fieldName, ir.RowIDs[i], ts) - panicOn(err) - if change { - changed = true - } - } else if len(ir.RowKeys) > 0 { //TODO (twg) these may not be reachable - change, err := action(indexName, column, fieldName, ir.RowKeys[i], ts) - panicOn(err) - if change { - changed = true - } - } - } - } -} -func (s *Sauron) handlePostImportRoaring(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - _ = body - _ = resp - _ = err - // Get index and field type to determine how to handle the - // import data. - indexName := mux.Vars(req)["index"] - fieldName := mux.Vars(req)["field"] - - irr := &pilosa.ImportRoaringRequest{} - err = proto.DefaultSerializer.Unmarshal(body, irr) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - //do i need the shard? - urlVars := mux.Vars(req) - shard, err := strconv.ParseUint(urlVars["shard"], 10, 64) - if err != nil { - panicOn(err) - } - for v, blob := range irr.Views { - _ = v //ignore the view for now - rit, err := roaring.NewRoaringIterator(blob) - panicOn(err) - s.sql.ImportRoaring(indexName, fieldName, shard, rit) - } -} - -//end handler -func (s *Sauron) ForwardHandler(w http.ResponseWriter, req *http.Request) (body []byte, resp *http.Response, err error) { - // TODO (twg) replace all the instances - // we need to buffer the body if we want to read it here and send it - // in the request. - body, err = ioutil.ReadAll(req.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // you can reassign the body if you need to parse it as multipart - req.Body = ioutil.NopCloser(bytes.NewReader(body)) - - // create a new url from the raw RequestURI sent by the client - url := fmt.Sprintf("%s%s", s.pilosaHttpURL, req.RequestURI) - - proxyReq, err := http.NewRequest(req.Method, url, bytes.NewReader(body)) - - // We may want to filter some headers, otherwise we could just use a shallow copy - // proxyReq.Header = req.Header - proxyReq.Header = make(http.Header) - for h, val := range req.Header { - proxyReq.Header[h] = val - } - - resp, err = http.DefaultClient.Do(proxyReq) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return - } - return -} - -// ImportRoaring must not be a bsi field..check higher up the stack -func (ps *PQLToSQL) ImportRoaring(index, field string, shard uint64, itr roaring.RoaringIterator) { - rb := roaring.NewBitmap() - for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { - rb.Put(itrKey, synthC) - } - for _, pos := range rb.Slice() { - row, column := pos/ps.ShardWidth, (shard*ps.ShardWidth)+(pos%ps.ShardWidth) - //ps.SetWithInt64Field(index, column, field, int64(row)) - ps.Set(index, column, field, int64(row), "") - } -} - -func (ps *PQLToSQL) isBSI(index, field string) (full string, isBSI bool) { - idx, ok := ps.i2f[index] - if !ok { - panic(fmt.Errorf("isBSI: no such index '%v'", index)) - } - - fld, ok := idx.Fields[field] - if !ok { - panic(fmt.Errorf("isBSI: no such field '%v'", field)) - } - if fld.Options.Type == "int" { - return dash2theta(index) + "λbsi", true - } - return dash2theta(index) + "λbits", false -} - -// Set is the basic bit operation and value operation as well, depending on the field type -func (ps *PQLToSQL) Set(index string, column uint64, field string, value interface{}, timestamp string) (bool, error) { - table, isBSI := ps.isBSI(index, field) - var sql string - if isBSI { - sql = fmt.Sprintf("INSERT INTO %v VALUES( '%v','%v', '%v')", table, field, column, value) - } else { - //if len(timestamp) == 0 { - // sql = fmt.Sprintf("INSERT INTO %v (field,row,column)VALUES( '%v','%v', '%v')", table, field, value, column) - // } else { - //TODO(twg) will need to make the timestamp as granular as the field is configured. - sql = fmt.Sprintf("INSERT INTO %v (field,row,column,timestamp)VALUES( '%v','%v', '%v','%v')", table, field, value, column, timestamp) - // } - } - - _, err := ps.DB.Exec(sql) - if err != nil { - if sqerr, ok := err.(*sqlite.Error); ok { - if sqerr.Code() == 2067 { //constraint error - return false, nil - } - } - } - panicOn(err) - return true, nil -} - -// Clear removes the specified column or value -func (ps *PQLToSQL) Clear(index string, column uint64, field string, value interface{}) (bool, error) { - table, isBSI := ps.isBSI(index, field) - var sql string - if isBSI { - sql = fmt.Sprintf("delete from %v where field='%v' and column='%v'", table, field, column) - } else { - sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' and column='%v'", table, field, value, column) - } - - res, err := ps.DB.Exec(sql) - panicOn(err) - r, err := res.RowsAffected() - panicOn(err) - if r == 0 { - return false, nil - } - return true, nil -} - -// ClearRow removes entire Row(feature) -func (ps *PQLToSQL) ClearRow(index string, field string, row interface{}) (bool, error) { - table, isBSI := ps.isBSI(index, field) - var sql string - if isBSI { - panic("bsi can't clear row allowed") - } else { - sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' ", table, field, row) - } - - r, err := ps.DB.Exec(sql) - panicOn(err) - n, err := r.RowsAffected() - return n > 0, err -} - -func sliceToString(slc []int64) (r string) { - if len(slc) == 0 { - return - } - for _, i := range slc { - r += fmt.Sprintf("%v,", i) - } - return r[:len(r)-1] -} - -const columnLabel = "col" - -func (ps *PQLToSQL) IssueQuery(req *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { - index := req.Index - q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() - if err != nil { - return nil, errors.Wrap(err, "PQLToSQL.IssueQuery parsing PQL") - } - var results pilosa.QueryResponse - for _, c := range q.Calls { - - if callIndex := c.CallIndex(); callIndex != "" { - index = callIndex - } - - // Handle the field arg. - switch strings.ToLower(c.Name) { - //handle the write queries - //TODO (twg) figure out sqlite changes() function to determine number of bits changed - case "set": - // Read colID. - colID, ok, err := c.UintArg("_" + columnLabel) - if err != nil { - return nil, fmt.Errorf("reading Set() column: %v", err) - } else if !ok { - return nil, fmt.Errorf("Set() column argument '%v' required", columnLabel) - } - ts, ok := c.Args["_timestamp"] - var timestamp string - if ok { - timestamp = strings.Replace(ts.(string), "T", " ", 1) - } - - field, err := c.FieldArg() - if err == nil { - rs, err := ps.Set(index, colID, field, c.Args[field], timestamp) - panicOn(err) - results.Results = append(results.Results, rs) - } else { - return nil, err - } - case "clear": - // Read colID. - colID, ok, err := c.UintArg("_" + columnLabel) - if err != nil { - return nil, fmt.Errorf("reading Clear() column: %v", err) - } else if !ok { - return nil, fmt.Errorf("Clear() column argument '%v' required", columnLabel) - } - - field, err := c.FieldArg() - if err == nil { - rs, err := ps.Clear(index, colID, field, c.Args[field]) - if err != nil { - return nil, fmt.Errorf("Clear() column: %v", err) - } - results.Results = append(results.Results, rs) - } else { - vv("c.FieldArg() returned '%v'", err) - } - case "clearrow": - field, err := c.FieldArg() - panicOn(err) - rs, err := ps.ClearRow(index, field, c.Args[field]) - panicOn(err) - results.Results = append(results.Results, rs) - case "store": - field, err := c.FieldArg() - panicOn(err) - rs, err := ps.Store(index, field, c.Args[field], c.Children) - panicOn(err) - results.Results = append(results.Results, rs) - default: //TODO (twg) move ToSQL into main package - rs, err := ps.readQuery(index, q) - panicOn(err) - results.Results = append(results.Results, rs) - } - } - return &results, nil -} -func (ps *PQLToSQL) handleBitmapCall(sql string) (*pilosa.Row, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - defer rows.Close() - columns := make([]uint64, 0) - for rows.Next() { - //really need a way to know what the results look like - var column uint64 - rows.Scan(&column) - columns = append(columns, column) - } - row := pilosa.NewRow(columns...) - return row, nil -} - -func (ps *PQLToSQL) handleIntCall(sql string) (count uint64, err error) { - rows, err := ps.DB.Query(sql) - if err != nil { - return 0, err - } - defer rows.Close() - for rows.Next() { - //really need a way to know what the results look like - rows.Scan(&count) - return - } - return -} - -type Table struct { -} - -func (ps *PQLToSQL) handleSignedRowCall(sql string) (pilosa.SignedRow, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - - var id uint64 - var pos []uint64 - var neg []uint64 - defer rows.Close() - for rows.Next() { - err := rows.Scan(&id) - if err != nil { - return pilosa.SignedRow{}, err - } - if id >= 0 { - pos = append(pos, id) - } else { - neg = append(neg, id) - } - } - err = rows.Err() - if err != nil { - return pilosa.SignedRow{}, err - } - posRow := pilosa.NewRow(pos...) - negRow := pilosa.NewRow(neg...) - return pilosa.SignedRow{Pos: posRow, Neg: negRow}, nil - -} -func (ps *PQLToSQL) handleStringArrayCall(sql string) ([]interface{}, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - rowKeys := make([]interface{}, 0) - var field string - var id string - defer rows.Close() - for rows.Next() { - err := rows.Scan(&field, &id) - if err != nil { - return nil, err - } - rowKeys = append(rowKeys, id) - } - err = rows.Err() - if err != nil { - return nil, err - } - return rowKeys, nil -} - -func (ps *PQLToSQL) handleIntArrayCall(sql string) (map[string][]interface{}, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - rowIDs := make([]interface{}, 0) - var id uint64 - var field string - defer rows.Close() - for rows.Next() { - err := rows.Scan(&field, &id) - if err != nil { - return nil, err - } - rowIDs = append(rowIDs, id) - } - err = rows.Err() - if err != nil { - return nil, err - } - m := map[string][]interface{}{"rows": rowIDs, "keys": []interface{}{}} - return m, nil -} -func (ps *PQLToSQL) handleGroupCountCall(sqlin string, index string, numFields int) (pilosa.SortedGroupCount, error) { - rows, err := ps.DB.Query(sqlin) - panicOn(err) - //groupCounts := make([]pilosa.GroupCount, 0) - var groupCounts pilosa.SortedGroupCount - defer rows.Close() - cols, err := rows.Columns() - if err != nil { - return nil, err - } - row := make([]interface{}, len(cols)) - for i := range cols { - row[i] = new(sql.RawBytes) - } - for rows.Next() { - // need a row*NumFields + cnt - //err := rows.Scan(&field, &row, &count) - groupCount := pilosa.GroupCount{} - - err := rows.Scan(row...) - if err != nil { - return nil, err - } - for i := 0; i < len(row)-1; i += 2 { - ptr := row[i].(*sql.RawBytes) - fieldName := string(*ptr) - fr := pilosa.FieldRow{Field: fieldName} - fi, err := ps.GetField(index, fieldName) - if err != nil { - return nil, err - } - ptr = row[i+1].(*sql.RawBytes) - v := string(*ptr) - if fi.Options.Keys { - fr.RowKey = v - - } else { - iv, err := strconv.Atoi(v) - if err != nil { - return nil, err - } - fr.RowID = uint64(iv) - } - - groupCount.Group = append(groupCount.Group, fr) - - } - ptr := row[len(row)-1].(*sql.RawBytes) - data := string(*ptr) //convert to string - i, err := strconv.ParseInt(data, 10, 64) - panicOn(err) - groupCount.Count = uint64(i) - groupCounts = append(groupCounts, groupCount) - - } - - err = rows.Err() - if err != nil { - return nil, err - } - return groupCounts, nil -} -func (ps *PQLToSQL) handlePairsFieldCall(sql, index string) (pilosa.PairsField, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - var count uint64 - var row string - defer rows.Close() - res := pilosa.PairsField{} - for rows.Next() { - err := rows.Scan(&row, &count) - if err != nil { - return pilosa.PairsField{}, err - } - pair := pilosa.Pair{Key: row, Count: count} - res.Pairs = append(res.Pairs, pair) - } - err = rows.Err() - if err != nil { - return pilosa.PairsField{}, err - } - return res, nil -} - -// special structures to deal with Extract. In the future I would like to use -// pilosa structs, but dealing with keys makes that complicated for now -// note the column is a string, but i only worry about the type when marshalling out for -// the sqlite part is typeless. Handling keys is almost seemless and not requiring translation -type extractColumns struct { - column string - rows map[int][]interface{} - parent *sauronExtractedIDMatrix -} - -func (eid extractColumns) MarshalJSON() ([]byte, error) { - buf := new(bytes.Buffer) - buf.WriteString(fmt.Sprintf(`{"column": %v,"rows":[`, eid.column)) - rows := make([][]interface{}, len(eid.parent.fields)) - for k, v := range eid.rows { - rows[k] = v - } - for i, grow := range rows { - if i != 0 { - buf.WriteByte(44) //write the comma - } - isKey := eid.parent.fields[i].isKey - buf.WriteString(`[`) - for d, row := range grow { - if d != 0 { - buf.WriteByte(44) //write the comma - } - if isKey { - buf.WriteString(fmt.Sprintf(`"%v"`, row)) - } else { - buf.WriteString(fmt.Sprintf(`%v`, row)) - } - - } - buf.WriteString(`]`) - } - - buf.WriteString(`]}`) - return buf.Bytes(), nil -} - -type extractFields struct { - Name string `json:"name"` - Type string `json:"type"` - isKey bool -} -type sauronExtractedIDMatrix struct { - columns []extractColumns - fields []extractFields -} - -func (eid sauronExtractedIDMatrix) MarshalJSON() ([]byte, error) { - buf := new(bytes.Buffer) - sort.Slice(eid.columns, func(i, j int) bool { - return eid.columns[i].column < eid.columns[j].column - }) - buf.WriteString(`{"columns":[`) - for i, col := range eid.columns { - b, err := col.MarshalJSON() - panicOn(err) - if i != 0 { - buf.WriteByte(44) //write the comma - } - buf.Write(b) - } - - buf.WriteString(`],"fields":[`) - for i, f := range eid.fields { - if i != 0 { - buf.WriteByte(44) - } - b, err := json.Marshal(f) - panicOn(err) - buf.Write(b) - } - buf.WriteString(`]}`) - return buf.Bytes(), nil -} - -func (ps *PQLToSQL) handleTableCall(sql, index string) (sauronExtractedIDMatrix, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - var field string - var row string - var column string - defer rows.Close() - col := make(map[string]map[string][]interface{}) - fidx := make(map[string]int) - results := sauronExtractedIDMatrix{} - counter := 0 - empty := make(map[string][]interface{}) - for rows.Next() { - err := rows.Scan(&field, &row, &column) - if err != nil { - return sauronExtractedIDMatrix{}, err - } - if field != "" { - _, ok := fidx[field] - if !ok { - fidx[field] = counter - counter++ - } - rows, ok := col[column] - if !ok { - rows = make(map[string][]interface{}) - } - rows[field] = append(rows[field], row) - //results.Columns = append(results.Columns, pilosa.ExtractedIDColumn{ColumnID: column, Rows: [][]uint64{}}) - col[column] = rows - } else { //constrow, or no filed info - col[column] = empty - } - } - err = rows.Err() - if err != nil { - return sauronExtractedIDMatrix{}, err - } - for c, rowMap := range col { - idc := extractColumns{rows: make(map[int][]interface{}), parent: &results} - idc.column = c - for fld, rows := range rowMap { - idx := fidx[fld] - idc.rows[idx] = rows - } - results.columns = append(results.columns, idc) - } - results.fields = make([]extractFields, len(fidx)) - for fieldName, idx := range fidx { - fi, _ := ps.GetField(index, fieldName) - // TODO (twg) hack need to figure out proper way to determine type - aType := "[]uint64" - if fi.Options.Keys { - aType = "[]string" - } - results.fields[idx] = extractFields{Name: fieldName, Type: aType, isKey: fi.Options.Keys} - } - - return results, nil -} - -func (ps *PQLToSQL) handleAggSQLCall(sql, index string) (pilosa.ValCount, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - defer rows.Close() - results := pilosa.ValCount{} - var val int64 - var count int64 - for rows.Next() { - err := rows.Scan(&val, &count) - if err != nil { - return pilosa.ValCount{}, err - } - results.Count = count - results.Val = val - break //shouldn't be anymore rows but being safe - } - err = rows.Err() - if err != nil { - return pilosa.ValCount{}, err - } - return results, nil -} -func (ps *PQLToSQL) handleBoolCall(sql, index string) (bool, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - defer rows.Close() - var val bool - for rows.Next() { - err := rows.Scan(&val) - if err != nil { - return false, err - } - break //shouldn't be anymore rows but being safe - } - err = rows.Err() - if err != nil { - return val, err - } - return val, nil -} - -func (ps *PQLToSQL) readQuery(index string, query *pql.Query) (interface{}, error) { - //Row(color=red) - sql, resultType, optional, err := ToSql(query.Calls, index) - panicOn(err) - switch resultType { - case BITMAP: - return ps.handleBitmapCall(sql) - case INT: - return ps.handleIntCall(sql) - case SIGNEDROW: - return ps.handleSignedRowCall(sql) - case ARRAY: - fieldName := optional[0].(string) - info, ok := ps.i2f[index] - if !ok { - return nil, errors.New(fmt.Sprintf("index not in schema %v", index)) - } - field, ok := info.Fields[fieldName] - if !ok { - return nil, errors.New(fmt.Sprintf("field not in schema %v", fieldName)) - } - if field.Options.Keys { - return ps.handleStringArrayCall(sql) - } else { - return ps.handleIntArrayCall(sql) - } - case GROUPCOUNT: - return ps.handleGroupCountCall(sql, index, optional[0].(int)) - case PAIRSFIELD: - return ps.handlePairsFieldCall(sql, index) - case TABLE: - return ps.handleTableCall(sql, index) - case AGGSQL: - return ps.handleAggSQLCall(sql, index) - case BOOL: - return ps.handleBoolCall(sql, index) - default: - vv("Unknown query Type %v", resultType) - } - return nil, nil -} - -type Int64Slice []int64 - -func (p Int64Slice) Len() int { return len(p) } -func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } -func (p Int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - -// on tables starting with ps.dbprefix will be returned. -func (ps *PQLToSQL) ListIndexFields() (tables []string) { - for indexName, index := range ps.i2f { - for fieldname, _ := range index.Fields { - - tables = append(tables, indexName+"/"+fieldname) - } - } - sort.Strings(tables) - return tables -} diff --git a/cmd/sauron/populate.sh b/cmd/sauron/populate.sh deleted file mode 100644 index 46e7dae67..000000000 --- a/cmd/sauron/populate.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -mod=20 -for i in `seq 100 1`;do -b=$(dc -e "$i $mod %p") -[[ $b == 0 ]] && mod=$(($mod+1)) -r=$(($b+30)) -payload="Set($i, j=$r)" -echo $payload -curl -XPOST "localhost:10101/index/i/query" -d "$payload" -done diff --git a/cmd/sauron/sample_schema.json b/cmd/sauron/sample_schema.json deleted file mode 100644 index 4dcc2c501..000000000 --- a/cmd/sauron/sample_schema.json +++ /dev/null @@ -1 +0,0 @@ -{"indexes":[{"name":"trait_store","createdAt":1595896639330112903,"options":{"keys":true,"trackExistence":true},"fields":[{"name":"aba","createdAt":1595896641512346986,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"ach_batch_recency","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_pass_thru_recency","createdAt":1595896640333428009,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_payment_recency","createdAt":1595896642230184052,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"bools","createdAt":1595896640151751600,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"bools-exists","createdAt":1595896640855732760,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"central_group","createdAt":1595896640036032355,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"custom_audiences","createdAt":1595896643284443107,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"days_since_last_logon","createdAt":1595896639332677618,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":1,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"db","createdAt":1595896639548206413,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"desktop_recency","createdAt":1595896642633713852,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"domestic_wire_recency","createdAt":1595896642264402807,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"external_transfer_recency","createdAt":1595896639332854139,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"fields","createdAt":1595896639552562813,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"funds_transfer_recency","createdAt":1595896640803192694,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"international_wire_recency","createdAt":1595896643511015744,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"mobile_remote_deposit_recency","createdAt":1595896641174868205,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"payroll_recency","createdAt":1595896642880416952,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"pfm_category_total_current_balance__401k_investment","createdAt":1595896641492935339,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__403b_investment","createdAt":1595896639553865549,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__529_investment","createdAt":1595896643478533291,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__any","createdAt":1595896642202343868,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__auto_loan","createdAt":1595896642260798767,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__brokerage_account","createdAt":1595896639550130078,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__certificate_of_deposit","createdAt":1595896643210723905,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__checking","createdAt":1595896640763548235,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":42,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__credit_card","createdAt":1595896639734800167,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__home_equity_loan","createdAt":1595896642644368353,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__ira_investment","createdAt":1595896642076795921,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__line_of_credit","createdAt":1595896642071432710,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__loan","createdAt":1595896643101755055,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__money_market","createdAt":1595896642574933677,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":36,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__mortgage","createdAt":1595896640934767155,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__personal_loan","createdAt":1595896641378825138,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__roth_ira_investment","createdAt":1595896642821349899,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__savings","createdAt":1595896639534597334,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__simple_ira","createdAt":1595896639952200646,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__student_loan","createdAt":1595896642329394180,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__taxable_investment","createdAt":1595896641406468398,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"phone_recency","createdAt":1595896639708002303,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_count__brokerage_account","createdAt":1595896642501159753,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_cd_or_share_certificate","createdAt":1595896643269910233,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_checking_or_share_draft","createdAt":1595896640044097611,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_credit_card","createdAt":1595896641500609762,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_indirect_auto_loan","createdAt":1595896643353298900,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_line_of_credit","createdAt":1595896641219946523,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_loan","createdAt":1595896641263557904,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_money_market","createdAt":1595896642506172994,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_new_auto_loan","createdAt":1595896640791087766,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_real_estate_loan","createdAt":1595896642616627035,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_savings_or_share","createdAt":1595896642313967229,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_credit_card","createdAt":1595896642254165546,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_loan","createdAt":1595896642870853670,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_checking_or_share_draft","createdAt":1595896639333642847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_credit_card","createdAt":1595896642761030299,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_savings_or_share","createdAt":1595896643227769780,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_vehicle_loan","createdAt":1595896642991171391,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_used_auto_loan","createdAt":1595896642865599530,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_basic_checking_or_share_draft","createdAt":1595896640110059946,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_cd_or_share_certificate","createdAt":1595896640174468431,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_credit_card","createdAt":1595896640192571502,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_indirect_auto_loan","createdAt":1595896640563441892,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896643543919820,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_line_of_credit","createdAt":1595896640188449923,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_loan","createdAt":1595896639761090179,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_money_market","createdAt":1595896642731261748,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_new_auto_loan","createdAt":1595896643143947051,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_checking_or_share_draft","createdAt":1595896640346982139,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_savings_or_share","createdAt":1595896641346726397,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_real_estate_loan","createdAt":1595896643351578694,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_checking_or_share_draft","createdAt":1595896640302899735,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_savings_or_share","createdAt":1595896642726551847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_savings_or_share","createdAt":1595896642961872488,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_credit_card","createdAt":1595896640022927417,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_loan","createdAt":1595896642352438551,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_checking_or_share_draft","createdAt":1595896640804808345,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_credit_card","createdAt":1595896641113696262,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_savings_or_share","createdAt":1595896640312498993,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_vehicle_loan","createdAt":1595896639886308884,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_used_auto_loan","createdAt":1595896642412302198,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__debit_card","createdAt":1595896643127876947,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__education_savings_or_share","createdAt":1595896639338093281,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__escrow_account","createdAt":1595896642969874721,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__health_savings_account","createdAt":1595896640570458731,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_line_of_credit","createdAt":1595896642592520668,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_loan","createdAt":1595896643091627581,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__insurance_product","createdAt":1595896639744476133,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__interest_only_legal_trust_account","createdAt":1595896643005843869,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__membership_share","createdAt":1595896641732585653,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__other_service","createdAt":1595896641807551269,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_401k","createdAt":1595896639333271302,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":11,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_403b","createdAt":1595896643165638942,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_529","createdAt":1595896641640608805,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira","createdAt":1595896639555618615,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira_cd_or_share_certificate","createdAt":1595896641504460031,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__small_business_checking_or_share_draft","createdAt":1595896639332189957,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__special_indirect_vehicle_loan","createdAt":1595896641880199490,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__student_education_loan","createdAt":1595896639551558423,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira","createdAt":1595896643372890708,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira_cd_or_share_certificate","createdAt":1595896641621211665,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__trust","createdAt":1595896640709548779,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__vacation_club_holiday_savings","createdAt":1595896642256997553,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_recency__brokerage_account","createdAt":1595896641955724697,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_cd_or_share_certificate","createdAt":1595896642377298176,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_checking_or_share_draft","createdAt":1595896641640736297,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_credit_card","createdAt":1595896641199968081,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_indirect_auto_loan","createdAt":1595896643314414987,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_line_of_credit","createdAt":1595896641576267257,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_loan","createdAt":1595896641355763147,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_money_market","createdAt":1595896639970740640,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_new_auto_loan","createdAt":1595896642844117593,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_real_estate_loan","createdAt":1595896642953871689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_savings_or_share","createdAt":1595896643325231815,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_credit_card","createdAt":1595896642698454456,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_loan","createdAt":1595896641951407002,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_checking_or_share_draft","createdAt":1595896642711316173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_credit_card","createdAt":1595896642147545663,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_savings_or_share","createdAt":1595896642364919307,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_vehicle_loan","createdAt":1595896639876538030,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_used_auto_loan","createdAt":1595896643527381053,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_basic_checking_or_share_draft","createdAt":1595896639874545938,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_cd_or_share_certificate","createdAt":1595896643147292944,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_credit_card","createdAt":1595896642605033674,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_indirect_auto_loan","createdAt":1595896641097157897,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896642523867033,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_line_of_credit","createdAt":1595896641979726767,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_loan","createdAt":1595896639741458231,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_money_market","createdAt":1595896642350195677,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_new_auto_loan","createdAt":1595896642852653224,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_checking_or_share_draft","createdAt":1595896641391887646,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_savings_or_share","createdAt":1595896643501816261,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_real_estate_loan","createdAt":1595896643015250459,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_checking_or_share_draft","createdAt":1595896640413965217,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_savings_or_share","createdAt":1595896642440765022,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_savings_or_share","createdAt":1595896643523788186,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_credit_card","createdAt":1595896641228514181,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_loan","createdAt":1595896642815200189,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_checking_or_share_draft","createdAt":1595896639648900382,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_credit_card","createdAt":1595896639596071703,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_savings_or_share","createdAt":1595896642447703684,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_vehicle_loan","createdAt":1595896639863224318,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_used_auto_loan","createdAt":1595896639337315659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__debit_card","createdAt":1595896641091084110,"options":{"type":"int","base":0,"bitDepth":1,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__education_savings_or_share","createdAt":1595896643243122929,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__escrow_account","createdAt":1595896642288649376,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__health_savings_account","createdAt":1595896639713687734,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_line_of_credit","createdAt":1595896640644328270,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_loan","createdAt":1595896643005117660,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__insurance_product","createdAt":1595896640080338115,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__interest_only_legal_trust_account","createdAt":1595896640710544689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__membership_share","createdAt":1595896640920327282,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__other_service","createdAt":1595896641868683380,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_401k","createdAt":1595896643214447056,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_403b","createdAt":1595896639331510173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_529","createdAt":1595896641093053659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira","createdAt":1595896640406186395,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira_cd_or_share_certificate","createdAt":1595896642473602012,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__small_business_checking_or_share_draft","createdAt":1595896640631139451,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__special_indirect_vehicle_loan","createdAt":1595896641787887178,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__student_education_loan","createdAt":1595896640496522510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira","createdAt":1595896643383850510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira_cd_or_share_certificate","createdAt":1595896642145329304,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__trust","createdAt":1595896640506789797,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__vacation_club_holiday_savings","createdAt":1595896642206110046,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"stop_payment_recency","createdAt":1595896640954094741,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"survey_2053c210_adfc_4e14_afd3_9abc68a70719","createdAt":1595896642841258189,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2679bd7e_cdb8_4c24_a1da_e904799382eb","createdAt":1595896642498273724,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2f336120_7130_492b_b116_d93d560baa0a","createdAt":1595896639333711539,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_36e229ff_c0a6_48ad_928e_19b5abf69f11","createdAt":1595896642987004213,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa","createdAt":1595896643012024691,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796","createdAt":1595896640184709609,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_587ae933_b8e3_49a2_a23d_989df9ad119f","createdAt":1595896642862965911,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab","createdAt":1595896642292355510,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949","createdAt":1595896640788795677,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_9cebb459_162e_4518_88f6_bc1cdd630cd0","createdAt":1595896642343098690,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_a05de6d3_6620_4eb2_b9a2_511936680b57","createdAt":1595896643389135517,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_b570363c_ba94_4a7a_9891_141a1befc42b","createdAt":1595896643120229702,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_c358d6e0_4116_4230_84ed_869b315e89a1","createdAt":1595896642202109079,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_cce773ce_14e8_4fae_90cd_213a821a3bc0","createdAt":1595896640040535589,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_e49df858_4903_40ac_bd9f_96f764940f2c","createdAt":1595896642346035396,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9","createdAt":1595896640643626147,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"tablet_recency","createdAt":1595896640414042095,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"user_id","createdAt":1595896640846590883,"options":{"type":"int","base":0,"bitDepth":21,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"zip_code","createdAt":1595896640534832822,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}}],"shardWidth":1048576}]} diff --git a/cmd/sauron/sauron_test.go b/cmd/sauron/sauron_test.go deleted file mode 100644 index 53cf786de..000000000 --- a/cmd/sauron/sauron_test.go +++ /dev/null @@ -1,396 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "encoding/hex" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strconv" - "testing" - - pb "github.com/golang/protobuf/proto" //nolint:staticcheck - pbuf "github.com/molecula/go-pilosa/v2/gopilosa_pbuf" - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/encoding/proto" - "github.com/pkg/errors" -) - -func TestSauron(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //remove the existing db file if present - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - - schemaJson := `{"indexes":[{"name":"scratch","createdAt":1611185870149882000,"options":{"keys":false,"trackExistence":true}, "fields":[{"name":"luminosity","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}}]}]}` - - client := &http.Client{} - t.Run("Schema", func(t *testing.T) { - _, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) - panicOn(err) - // verify that we can fetch it back - resp, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp.Body) - panicOn(err) - sbody := string(body) - if sbody[:62] != schemaJson[:62] { - fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) - panic("unexpected schema back") - } - }) - // add a set field "color" - //POST localhost:10101/index/scratch/field/color - t.Run("Create Non-Keyed SetField", func(t *testing.T) { - resp, err := client.Post(url+"/index/scratch/field/rating", "application/text", bytes.NewBuffer(nil)) - panicOn(err) - if resp.StatusCode != 200 { - panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) - } - }) - t.Run("Create Keyed SetField", func(t *testing.T) { - resp, err := client.Post(url+"/index/scratch/field/color", "application/text", bytes.NewBuffer([]byte(`{"options": {"keys": true}}`))) - panicOn(err) - if resp.StatusCode != 200 { - panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) - } - }) - t.Run("Create Time Field", func(t *testing.T) { - resp, err := client.Post(url+"/index/scratch/field/event", "application/text", bytes.NewBuffer([]byte(`{"options": { "type": "time", "timeQuantum": "YMDH" }}`))) - panicOn(err) - if resp.StatusCode != 200 { - panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) - } - }) - - // set some bits - - // set some bits - - t.Run("Set", func(t *testing.T) { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) - panicOn(err) - // test for dups - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) - panicOn(err) - // - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=red)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=green)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=green)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=green)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=green)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=yellow)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=yellow)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=yellow)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=yellow)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=orange)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=orange)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=orange)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=orange)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, luminosity=1)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, luminosity=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, luminosity=4)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, luminosity=5)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, luminosity=4)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, luminosity=3)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, luminosity=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, luminosity=1)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, rating=1)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(10, rating=1)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, rating=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, rating=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, rating=3)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, rating=4)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, rating=4)`)) - panicOn(err) - - //time fields - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T01:00)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T02:00)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, event=1,2021-02-05T02:05)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, event=1,2021-02-05T03:00)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, event=1,2021-02-05T04:00)`)) - panicOn(err) - }) - t.Run("Clear", func(t *testing.T) { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Clear(1, color=purple)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`ClearRow(color=purple)`)) - panicOn(err) - }) - t.Run("Store", func(t *testing.T) { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Store(Row(color=red), color=purple)`)) - panicOn(err) - }) - // verify that we can fetch it back - var schema pilosa.Schema - t.Run("Fetch Schema", func(t *testing.T) { - resp, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp.Body) - panicOn(err) - err = json.Unmarshal(body, &schema) - if err != nil { - panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) - } - }) - t.Run("Import Roaring", func(t *testing.T) { - //sets 12 bits in shard 0 - roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - idx := schema.Indexes[0] - var fld *pilosa.FieldInfo - for _, f := range idx.Fields { - if f.Name == "rating" { - fld = f - } - } - if fld == nil { - panic("field color not found") - } - msg := pilosa.ImportRoaringRequest{ - IndexCreatedAt: idx.CreatedAt, - FieldCreatedAt: fld.CreatedAt, - Clear: false, - Views: map[string][]byte{ - "": roaringData, - }, - UpdateExistence: true, - } - ser := proto.Serializer{} - data, err := ser.Marshal(&msg) - if err != nil { - t.Fatal(err) - } - furl := url + "/index/scratch/field/rating/import-roaring/0" - req, err := http.NewRequest("POST", furl, bytes.NewBuffer(data)) - panicOn(err) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - - // maybe need to Accept: "application/x-protobuf" - resp, err := client.Do(req) - panicOn(err) - _ = resp - }) - t.Run("Read Query", func(t *testing.T) { - for _, tst := range GetTests() { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(tst.Pql)) - panicOn(err) - } - }) -} - -func makeImportRequest(field *FieldInfo2, shard uint64, rows, columns []uint64, clear bool) (path string, data []byte, err error) { - msg := &pbuf.ImportRequest{ - Index: field.GetIndexName(), - IndexCreatedAt: field.GetIndexCreatedAt(), - Field: field.GetName(), - FieldCreatedAt: field.GetCreatedAt(), - Shard: shard, - RowIDs: rows, - ColumnIDs: columns, - } - data, err = pb.Marshal(msg) - if err != nil { - return "", nil, errors.Wrap(err, "marshaling Import to protobuf") - } - path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) - return path, data, nil -} -func makeImportKeyRequest(field *FieldInfo2, shard uint64, rows []string, columns []uint64, clear bool) (path string, data []byte, err error) { - msg := &pbuf.ImportRequest{ - Index: field.GetIndexName(), - IndexCreatedAt: field.GetIndexCreatedAt(), - Field: field.GetName(), - FieldCreatedAt: field.GetCreatedAt(), - Shard: shard, - RowKeys: rows, - ColumnIDs: columns, - } - data, err = pb.Marshal(msg) - if err != nil { - return "", nil, errors.Wrap(err, "marshaling Import to protobuf") - } - path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) - return path, data, nil -} - -func makeImportValueRequest(field *FieldInfo2, shard uint64, values []int64, columns []uint64, clear bool) (path string, data []byte, err error) { - msg := &pbuf.ImportValueRequest{ - Index: field.GetIndexName(), - IndexCreatedAt: field.GetIndexCreatedAt(), - Field: field.GetName(), - FieldCreatedAt: field.GetCreatedAt(), - Shard: shard, - Values: values, - ColumnIDs: columns, - } - data, err = pb.Marshal(msg) - if err != nil { - return "", nil, errors.Wrap(err, "marshaling Import to protobuf") - } - path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) - return path, data, nil -} -func TestSauronImport(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //remove the existing db file if present - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - schemaJson := `{ - "indexes": [{ - "name": "scratch", - "createdAt": 1611185870149882000, - "options": { - "keys": false, - "trackExistence": true - }, - "fields": [{ - "name": "bsi", - "createdAt": 1595896639332730413, - "options": { - "type": "int", - "base": 0, - "bitDepth": 31, - "min": -9223372036854775808, - "max": 9223372036854775807, - "keys": false, - "foreignIndex": "" - } - }, { - "name": "decimal", - "createdAt": 1613177295654228860, - "options": { - "type": "decimal", - "base": 0, - "scale": 1, - "bitDepth": 0, - "min": -922337203685477580.8, - "max": 922337203685477580.7, - "keys": false - } - },{ - "name": "setkeyfield", - "createdAt": 1613345793598357200, - "options": { - "type": "set", - "cacheType": "ranked", - "cacheSize": 50000, - "keys": true - } - },{ - "name": "setfield", - "createdAt": 1613345793598357200, - "options": { - "type": "set", - "cacheType": "ranked", - "cacheSize": 50000, - "keys": false - } - }] - }] -}` - client := &http.Client{} - _, err = client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) - panicOn(err) - - t.Run("basic row/column", func(t *testing.T) { - f, err := proxy.sql.GetField("scratch", "setfield") - panicOn(err) - clear := false - cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} - rows := []uint64{0, 0, 0, 1, 1, 1, 2, 2, 2} - shard := uint64(0) - path, payload, err := makeImportRequest(f, shard, rows, cols, clear) - _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) - panicOn(err) - }) - t.Run("basic rowkey/column", func(t *testing.T) { - f, err := proxy.sql.GetField("scratch", "setkeyfield") - panicOn(err) - clear := false - cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} - rows := []string{"red", "red", "red", "blue", "blue", "blue", "green", "green", "green"} - shard := uint64(0) - path, payload, err := makeImportKeyRequest(f, shard, rows, cols, clear) - _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) - panicOn(err) - }) - t.Run("basic import values", func(t *testing.T) { - f, err := proxy.sql.GetField("scratch", "bsi") - panicOn(err) - clear := false - cols := []uint64{0, 1, 2, 3, 4} - values := []int64{40, 30, 20, 10, 5} - shard := uint64(0) - path, payload, err := makeImportValueRequest(f, shard, values, cols, clear) - _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) - panicOn(err) - }) - -} diff --git a/cmd/sauron/schema_test.go b/cmd/sauron/schema_test.go deleted file mode 100644 index 23f5e924b..000000000 --- a/cmd/sauron/schema_test.go +++ /dev/null @@ -1,353 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strings" - "testing" - - "github.com/pilosa/pilosa/v2" -) - -func NewSauronTestConfig(t *testing.T) *SauronConfig { - - bind := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) - bindGRPC := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) - gossipPort := fmt.Sprintf("%d", GetAvailPort()) - dataDir, err := ioutil.TempDir("", "pilosa-sauron-target-*") - panicOn(err) - - return &SauronConfig{ - DatabaseName: strings.ToLower(t.Name()), - Bind: bind, - BindGRPC: bindGRPC, - GossipPort: gossipPort, - DataDir: dataDir, - } -} - -func TestProxyCanHandlePostSchema(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //start with - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - - // prep for test by cleaning up leftovers from any old run. - //panicOn(proxy.sql.DropTablesWithPrefix(t.Name())) - - //schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } }, { "name ": "color", "options ": {} } ] }] }` - /* - schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } } ] }] }` - - - vv("Posting err = '%v'", schemaJson) - resp, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) - if resp.StatusCode != http.StatusNoContent { - panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) - } - // verify that we can fetch it back - resp2, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp2.Body) - panicOn(err) - sbody := string(body) - vv("body = '%v'", sbody) - if sbody[:62] != schemaJson[:62] { - fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) - panic("unxpected schema back") - } - */ - // load large schema too - client := &http.Client{} - schm, err := ioutil.ReadFile("sample_schema.json") - panicOn(err) - - resp, err := client.Post(url+"/schema", "application/json", bytes.NewBuffer(schm)) - panicOn(err) - if resp.StatusCode != http.StatusNoContent { - panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) - } - - // verify all tables expected are present - obs := make(map[string]bool) - for i, table := range proxy.sql.ListIndexFields() { - _ = i - obs[table] = true - if !expectedTables[table] { - panic(fmt.Sprintf("observed table '%v' but was not expected", table)) - } - } - for table := range expectedTables { - if !obs[table] { - panic(fmt.Sprintf("expected table '%v' but was not observed", table)) - } - } - -} - -func TestProxyCanCreateIndexFieldViaPost(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //start with - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - client := &http.Client{} - //create non-keyed or "simple" index - resp, err := client.Post(url+"/index/simple", "application/json", bytes.NewBuffer(nil)) - panicOn(err) - if resp.StatusCode != http.StatusOK { - panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) - } - - fields := []struct { - name string - options []byte - }{ - {name: "set", options: []byte{}}, - {name: "keyset", options: []byte{}}, - {name: "bsi", options: []byte{}}, - } - for _, field := range fields { - //create set field - resp, err = client.Post(fmt.Sprintf("%v/index/simple/field/%v", url, field.name), "application/json", bytes.NewBuffer(field.options)) - panicOn(err) - if resp.StatusCode != http.StatusOK { - panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) - } - } - // verify that we can fetch it back - resp2, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp2.Body) - panicOn(err) - var schema pilosa.Schema - err = json.Unmarshal(body, &schema) - if err != nil { - panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) - } - - expected := map[string]map[string]bool{ - "simple": {"bsi": true, "set": true, "keyset": true}, - } - for _, i := range schema.Indexes { - ll, ok := expected[i.Name] - if !ok { - panic(fmt.Sprintf("expected index not present: %v", i.Name)) - } - for _, f := range i.Fields { - _, ok := ll[f.Name] - if !ok { - panic(fmt.Sprintf("expected field '%v' not present in index'%v'", f.Name, i.Name)) - } - } - } -} - -var expectedTables = map[string]bool{ - "trait_store/aba": true, - "trait_store/ach_batch_recency": true, - "trait_store/ach_pass_thru_recency": true, - "trait_store/ach_payment_recency": true, - "trait_store/bools": true, - "trait_store/bools-exists": true, - "trait_store/central_group": true, - "trait_store/custom_audiences": true, - "trait_store/days_since_last_logon": true, - "trait_store/db": true, - "trait_store/desktop_recency": true, - "trait_store/domestic_wire_recency": true, - "trait_store/external_transfer_recency": true, - "trait_store/fields": true, - "trait_store/funds_transfer_recency": true, - "trait_store/international_wire_recency": true, - "trait_store/mobile_remote_deposit_recency": true, - "trait_store/payroll_recency": true, - "trait_store/pfm_category_total_current_balance__401k_investment": true, - "trait_store/pfm_category_total_current_balance__403b_investment": true, - "trait_store/pfm_category_total_current_balance__529_investment": true, - "trait_store/pfm_category_total_current_balance__any": true, - "trait_store/pfm_category_total_current_balance__auto_loan": true, - "trait_store/pfm_category_total_current_balance__brokerage_account": true, - "trait_store/pfm_category_total_current_balance__certificate_of_deposit": true, - "trait_store/pfm_category_total_current_balance__checking": true, - "trait_store/pfm_category_total_current_balance__credit_card": true, - "trait_store/pfm_category_total_current_balance__home_equity_loan": true, - "trait_store/pfm_category_total_current_balance__ira_investment": true, - "trait_store/pfm_category_total_current_balance__line_of_credit": true, - "trait_store/pfm_category_total_current_balance__loan": true, - "trait_store/pfm_category_total_current_balance__money_market": true, - "trait_store/pfm_category_total_current_balance__mortgage": true, - "trait_store/pfm_category_total_current_balance__personal_loan": true, - "trait_store/pfm_category_total_current_balance__roth_ira_investment": true, - "trait_store/pfm_category_total_current_balance__savings": true, - "trait_store/pfm_category_total_current_balance__simple_ira": true, - "trait_store/pfm_category_total_current_balance__student_loan": true, - "trait_store/pfm_category_total_current_balance__taxable_investment": true, - "trait_store/phone_recency": true, - "trait_store/product_count__brokerage_account": true, - "trait_store/product_count__commercial_cd_or_share_certificate": true, - "trait_store/product_count__commercial_checking_or_share_draft": true, - "trait_store/product_count__commercial_credit_card": true, - "trait_store/product_count__commercial_indirect_auto_loan": true, - "trait_store/product_count__commercial_line_of_credit": true, - "trait_store/product_count__commercial_loan": true, - "trait_store/product_count__commercial_money_market": true, - "trait_store/product_count__commercial_new_auto_loan": true, - "trait_store/product_count__commercial_real_estate_loan": true, - "trait_store/product_count__commercial_savings_or_share": true, - "trait_store/product_count__commercial_secured_credit_card": true, - "trait_store/product_count__commercial_secured_loan": true, - "trait_store/product_count__commercial_special_checking_or_share_draft": true, - "trait_store/product_count__commercial_special_credit_card": true, - "trait_store/product_count__commercial_special_savings_or_share": true, - "trait_store/product_count__commercial_special_vehicle_loan": true, - "trait_store/product_count__commercial_used_auto_loan": true, - "trait_store/product_count__consumer_basic_checking_or_share_draft": true, - "trait_store/product_count__consumer_cd_or_share_certificate": true, - "trait_store/product_count__consumer_credit_card": true, - "trait_store/product_count__consumer_indirect_auto_loan": true, - "trait_store/product_count__consumer_interest_bearing_checking_or_share_draft": true, - "trait_store/product_count__consumer_line_of_credit": true, - "trait_store/product_count__consumer_loan": true, - "trait_store/product_count__consumer_money_market": true, - "trait_store/product_count__consumer_new_auto_loan": true, - "trait_store/product_count__consumer_premium_checking_or_share_draft": true, - "trait_store/product_count__consumer_premium_savings_or_share": true, - "trait_store/product_count__consumer_real_estate_loan": true, - "trait_store/product_count__consumer_restricted_checking_or_share_draft": true, - "trait_store/product_count__consumer_restricted_savings_or_share": true, - "trait_store/product_count__consumer_savings_or_share": true, - "trait_store/product_count__consumer_secured_credit_card": true, - "trait_store/product_count__consumer_secured_loan": true, - "trait_store/product_count__consumer_special_checking_or_share_draft": true, - "trait_store/product_count__consumer_special_credit_card": true, - "trait_store/product_count__consumer_special_savings_or_share": true, - "trait_store/product_count__consumer_special_vehicle_loan": true, - "trait_store/product_count__consumer_used_auto_loan": true, - "trait_store/product_count__debit_card": true, - "trait_store/product_count__education_savings_or_share": true, - "trait_store/product_count__escrow_account": true, - "trait_store/product_count__health_savings_account": true, - "trait_store/product_count__home_equity_line_of_credit": true, - "trait_store/product_count__home_equity_loan": true, - "trait_store/product_count__insurance_product": true, - "trait_store/product_count__interest_only_legal_trust_account": true, - "trait_store/product_count__membership_share": true, - "trait_store/product_count__other_service": true, - "trait_store/product_count__plan_401k": true, - "trait_store/product_count__plan_403b": true, - "trait_store/product_count__plan_529": true, - "trait_store/product_count__roth_ira": true, - "trait_store/product_count__roth_ira_cd_or_share_certificate": true, - "trait_store/product_count__small_business_checking_or_share_draft": true, - "trait_store/product_count__special_indirect_vehicle_loan": true, - "trait_store/product_count__student_education_loan": true, - "trait_store/product_count__traditional_ira": true, - "trait_store/product_count__traditional_ira_cd_or_share_certificate": true, - "trait_store/product_count__trust": true, - "trait_store/product_count__vacation_club_holiday_savings": true, - "trait_store/product_recency__brokerage_account": true, - "trait_store/product_recency__commercial_cd_or_share_certificate": true, - "trait_store/product_recency__commercial_checking_or_share_draft": true, - "trait_store/product_recency__commercial_credit_card": true, - "trait_store/product_recency__commercial_indirect_auto_loan": true, - "trait_store/product_recency__commercial_line_of_credit": true, - "trait_store/product_recency__commercial_loan": true, - "trait_store/product_recency__commercial_money_market": true, - "trait_store/product_recency__commercial_new_auto_loan": true, - "trait_store/product_recency__commercial_real_estate_loan": true, - "trait_store/product_recency__commercial_savings_or_share": true, - "trait_store/product_recency__commercial_secured_credit_card": true, - "trait_store/product_recency__commercial_secured_loan": true, - "trait_store/product_recency__commercial_special_checking_or_share_draft": true, - "trait_store/product_recency__commercial_special_credit_card": true, - "trait_store/product_recency__commercial_special_savings_or_share": true, - "trait_store/product_recency__commercial_special_vehicle_loan": true, - "trait_store/product_recency__commercial_used_auto_loan": true, - "trait_store/product_recency__consumer_basic_checking_or_share_draft": true, - "trait_store/product_recency__consumer_cd_or_share_certificate": true, - "trait_store/product_recency__consumer_credit_card": true, - "trait_store/product_recency__consumer_indirect_auto_loan": true, - "trait_store/product_recency__consumer_interest_bearing_checking_or_share_draft": true, - "trait_store/product_recency__consumer_line_of_credit": true, - "trait_store/product_recency__consumer_loan": true, - "trait_store/product_recency__consumer_money_market": true, - "trait_store/product_recency__consumer_new_auto_loan": true, - "trait_store/product_recency__consumer_premium_checking_or_share_draft": true, - "trait_store/product_recency__consumer_premium_savings_or_share": true, - "trait_store/product_recency__consumer_real_estate_loan": true, - "trait_store/product_recency__consumer_restricted_checking_or_share_draft": true, - "trait_store/product_recency__consumer_restricted_savings_or_share": true, - "trait_store/product_recency__consumer_savings_or_share": true, - "trait_store/product_recency__consumer_secured_credit_card": true, - "trait_store/product_recency__consumer_secured_loan": true, - "trait_store/product_recency__consumer_special_checking_or_share_draft": true, - "trait_store/product_recency__consumer_special_credit_card": true, - "trait_store/product_recency__consumer_special_savings_or_share": true, - "trait_store/product_recency__consumer_special_vehicle_loan": true, - "trait_store/product_recency__consumer_used_auto_loan": true, - "trait_store/product_recency__debit_card": true, - "trait_store/product_recency__education_savings_or_share": true, - "trait_store/product_recency__escrow_account": true, - "trait_store/product_recency__health_savings_account": true, - "trait_store/product_recency__home_equity_line_of_credit": true, - "trait_store/product_recency__home_equity_loan": true, - "trait_store/product_recency__insurance_product": true, - "trait_store/product_recency__interest_only_legal_trust_account": true, - "trait_store/product_recency__membership_share": true, - "trait_store/product_recency__other_service": true, - "trait_store/product_recency__plan_401k": true, - "trait_store/product_recency__plan_403b": true, - "trait_store/product_recency__plan_529": true, - "trait_store/product_recency__roth_ira": true, - "trait_store/product_recency__roth_ira_cd_or_share_certificate": true, - "trait_store/product_recency__small_business_checking_or_share_draft": true, - "trait_store/product_recency__special_indirect_vehicle_loan": true, - "trait_store/product_recency__student_education_loan": true, - "trait_store/product_recency__traditional_ira": true, - "trait_store/product_recency__traditional_ira_cd_or_share_certificate": true, - "trait_store/product_recency__trust": true, - "trait_store/product_recency__vacation_club_holiday_savings": true, - "trait_store/stop_payment_recency": true, - "trait_store/survey_2053c210_adfc_4e14_afd3_9abc68a70719": true, - "trait_store/survey_2679bd7e_cdb8_4c24_a1da_e904799382eb": true, - "trait_store/survey_2f336120_7130_492b_b116_d93d560baa0a": true, - "trait_store/survey_36e229ff_c0a6_48ad_928e_19b5abf69f11": true, - "trait_store/survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa": true, - "trait_store/survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796": true, - "trait_store/survey_587ae933_b8e3_49a2_a23d_989df9ad119f": true, - "trait_store/survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab": true, - "trait_store/survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949": true, - "trait_store/survey_9cebb459_162e_4518_88f6_bc1cdd630cd0": true, - "trait_store/survey_a05de6d3_6620_4eb2_b9a2_511936680b57": true, - "trait_store/survey_b570363c_ba94_4a7a_9891_141a1befc42b": true, - "trait_store/survey_c358d6e0_4116_4230_84ed_869b315e89a1": true, - "trait_store/survey_cce773ce_14e8_4fae_90cd_213a821a3bc0": true, - "trait_store/survey_e49df858_4903_40ac_bd9f_96f764940f2c": true, - "trait_store/survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9": true, - "trait_store/tablet_recency": true, - "trait_store/user_id": true, - "trait_store/zip_code": true, -} diff --git a/cmd/sauron/sqlgen_test.go b/cmd/sauron/sqlgen_test.go deleted file mode 100644 index b9d11a728..000000000 --- a/cmd/sauron/sqlgen_test.go +++ /dev/null @@ -1,260 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/v2/pql" - "github.com/shurcooL/go-goon" -) - -/* -sqlite db this test validated this against -create table bits (field ,row , column , timestamp); - create table λbsi (field string,column bigint , val bigint); - CREATE VIEW columns as - select distinct column from λbits - UNION - select distinct column from λbsi; - - INSERT INTO λbits VALUES( 'color','red',1); - INSERT INTO λbits VALUES( 'color','red',2); - INSERT INTO λbits VALUES( 'color','red',3); - INSERT INTO λbits VALUES( 'color','red',4); - INSERT INTO λbits VALUES( 'color','red',5); - INSERT INTO λbits VALUES( 'color','green',2); - INSERT INTO λbits VALUES( 'color','green',4); - INSERT INTO λbits VALUES( 'color','green',6); - INSERT INTO λbits VALUES( 'color','green',8); - INSERT INTO λbits VALUES( 'color','yellow',1); - INSERT INTO λbits VALUES( 'color','yellow',3); - INSERT INTO λbits VALUES( 'color','yellow',5); - INSERT INTO λbits VALUES( 'color','yellow',7); - INSERT INTO λbits VALUES( 'color','orange',5); - INSERT INTO λbits VALUES( 'color','orange',6); - INSERT INTO λbits VALUES( 'color','orange',7); - INSERT INTO λbits VALUES( 'color','orange',8); - - INSERT INTO λbits VALUES( 'rating',1,8); - INSERT INTO λbits VALUES( 'rating',1,7); - INSERT INTO λbits VALUES( 'rating',1,10); - INSERT INTO λbits VALUES( 'rating',2,6); - INSERT INTO λbits VALUES( 'rating',2,5); - INSERT INTO λbits VALUES( 'rating',3,4); - INSERT INTO λbits VALUES( 'rating',4,3); - INSERT INTO λbits VALUES( 'rating',4,2); - - INSERT INTO λbsi VALUES( 'luminosity',1, 1); - INSERT INTO λbsi VALUES( 'luminosity',2, 2); - INSERT INTO λbsi VALUES( 'luminosity',3, 4); - INSERT INTO λbsi VALUES( 'luminosity',4, 5); - INSERT INTO λbsi VALUES( 'luminosity',5, 4); - INSERT INTO λbsi VALUES( 'luminosity',6, 3); - INSERT INTO λbsi VALUES( 'luminosity',7, 2); - INSERT INTO λbsi VALUES( 'luminosity',8, 1); -*/ -func GetTests() []struct { - Pql string - Sql string -} { - return []struct { - Pql string - Sql string - }{ - { - Pql: "Row(rating=0)", - Sql: `select distinct column from λbits where field="rating" AND row="0"`, - }, - { - Pql: "Row(color=red)", - Sql: `select distinct column from λbits where field="color" AND row="red"`, - }, - { - Pql: "Row(luminosity>2)", - Sql: `select distinct column from λbsi where field="luminosity" and val>2`, - }, - { - Pql: "Row(luminosity<2)", - Sql: `select distinct column from λbsi where field="luminosity" and val<2`, - }, - { - Pql: "Row(luminosity==4)", - Sql: `select distinct column from λbsi where field="luminosity" and val=4`, - }, - { - Pql: "Row(luminosity<=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val<=4`, - }, - { - Pql: "Row(luminosity>=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val>=4`, - }, - { - Pql: "Row(luminosity!=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val!=4`, - }, - { - Pql: "Row(2<=luminosity<=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val>=2 and val<=4`, - }, - { - Pql: "Row(22 and val<=4`, - }, - { - Pql: "Row(22 and val<4`, - }, - { - Pql: `Row(event=1, from='2021-02-01T00:00', to='2021-02-05T03:00')`, - Sql: `select distinct column from λbits where field="event" AND row="1" AND timestamp >= "2021-02-01 00:00" AND timestamp < "2021-02-05 03:00"`, - }, - { - Pql: "Count(row(color=red))", - Sql: `select count(*) from( select distinct column from λbits where field="color" AND row="red" )`, - }, - { - Pql: `Intersect(Row(color=red),Row(color=green))`, - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: "Count(Union(Row(color=red),Row(color=green)))", - Sql: `select count(*) from( select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") )`, - }, - { - Pql: "Union(Row(color=red),Row(color=green))", - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: "Difference(Row(color=red),Row(color=green))", - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" except select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: `All()`, - Sql: `select column from λcolumns`, - }, - { - Pql: "Not(Row(color=red))", - Sql: `select column from λcolumns except select distinct column from λbits where field="color" AND row="red"`, - }, - { - Pql: "Xor(Row(color=red),Row(color=green))", - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") except select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: "Distinct(field=luminosity)", - Sql: `select distinct val from λbsi where field="luminosity"`, - }, - { - Pql: "Distinct(Row(color=red),field=luminosity)", - Sql: `select distinct val from λbsi where field="luminosity" AND column in (select distinct column from λbits where field="color" AND row="red")`, - }, - { - Pql: "Distinct(Intersect(Row(color=red),Row(color=green)),field=luminosity)", - Sql: `select distinct val from λbsi where field="luminosity" AND column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green"))`, - }, - { - Pql: "Rows(color)", - Sql: `select distinct field, row from λbits where field="color" order by row`, - }, - { - Pql: "Rows(color, limit=10)", - Sql: `select distinct field, row from λbits where field="color" order by row limit 10`, - }, - { - Pql: "Rows(color, column=1)", - Sql: `select distinct field, row from λbits where field="color" and column='1' order by row`, - }, - // { // TODO (twg) Not supported by this test, can't reliably get key ordering - // Pql: "Rows(color, previous=green )", - // Sql: `select distinct field, row from λbits where field="color" and row>'green' order by row`, - // }, - { - Pql: "GroupBy(Rows(color))", - Sql: `select f1.field, f1.row,count(*) as cnt from λbits f1 where f1.field='color' group by f1.field,f1.row order by cnt desc`, - }, - { - Pql: "GroupBy(Rows(color),Rows(rating))", - Sql: `select f1.field, f1.row,f2.field, f2.row,count(*) as cnt from λbits f1 inner join λbits f2 on f1.column = f2.column where f1.field='color'and f2.field='rating' group by f1.field,f1.row,f2.field,f2.row order by cnt desc`, - }, - { - Pql: "TopN(color)", - Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, - }, - { - Pql: "TopN(color,n=2)", - Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc limit 2`, - }, - { - Pql: "TopN(color,Row(color=red),n=2)", - Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="color" AND row="red") group by row order by cnt desc limit 2`, - }, - { - Pql: "Topk(color)", - Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, - }, - { - Pql: "TopK(color, k=2, filter=Row(rating=1))", - Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="rating" AND row="1") group by row order by cnt desc limit 2`, - }, - { - Pql: `Min(field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity")`, - }, - { - Pql: `Max(field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select max(val) from λbsi where field="luminosity")`, - }, - { - Pql: `Min(Row(color=orange),field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select distinct column from λbits where field="color" AND row="orange")) and column in (select distinct column from λbits where field="color" AND row="orange")`, - }, - { - Pql: `Min(Intersect(Row(color=red),Row(color=orange)),field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))) and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))`, - }, - { - Pql: `Sum(field="luminosity")`, - Sql: `select sum(val),count(*) from λbsi where field="luminosity"`, - }, - { - Pql: `ConstRow(columns=[1,3,5])`, - Sql: `select column1 as column from (values (1),(3),(5))`, - }, - { - Pql: `UnionRows(Rows(color))`, - Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, - }, - { - Pql: `UnionRows(Rows(color),Rows(rating))`, - Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, - }, - { - Pql: `IncludesColumn(Row(color=red), column=1)`, - Sql: `select count(*)>0 from (select column from(select distinct column from λbits where field="color" AND row="red") where column='1')`, - }, - { - Pql: `Extract(Row(color=red),Rows(color),Rows(rating))`, - Sql: `select sq.field, sq.row, b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row where b.column in (select distinct column from λbits where field="color" AND row="red")`, - }, - { - Pql: `Extract(ConstRow(columns=[1,2,3]))`, - Sql: `select '' as field, '' as row,column from (select column1 as column from (values (1),(2),(3)))`, - }, - } -} -func TestParse(t *testing.T) { - - for _, tst := range GetTests() { - - q, err := pql.NewParser(strings.NewReader(tst.Pql)).Parse() - panicOn(err) - sql, _, _, err := ToSql(q.Calls, "") - panicOn(err) - if sql != tst.Sql { - vv("\npql:%v\ngenerated:\n%v\nexpected:\n%v\n", tst.Pql, sql, tst.Sql) - goon.Dump(q) - } - - } -} diff --git a/cmd/sauron/test_sqlite.sh b/cmd/sauron/test_sqlite.sh deleted file mode 100755 index d0ab85b2f..000000000 --- a/cmd/sauron/test_sqlite.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -db="poc2.db" -sqlite3 ${db} < 0 { - frames := runtime.CallersFrames(pc[:n]) - for i := 0; i <= target; i++ { - contender, more := frames.Next() - if i == target { - f = contender - } - if !more { - break - } - } - } - return f.Function -} diff --git a/go.mod b/go.mod index dcd6b603a..a9a8ef35d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d - github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 + github.com/DataDog/datadog-go v2.2.0+incompatible + github.com/OneOfOne/xxhash v1.2.5 // indirect + github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/beevik/ntp v0.3.0 github.com/benbjohnson/immutable v0.3.0 github.com/cespare/xxhash v1.1.0 @@ -15,26 +17,30 @@ require ( github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 - github.com/gogo/protobuf v1.2.1 + github.com/gogo/protobuf v1.3.1 github.com/golang/protobuf v1.4.2 github.com/google/go-cmp v0.5.2 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect - github.com/gorilla/handlers v1.3.0 - github.com/gorilla/mux v1.7.0 - github.com/hashicorp/memberlist v0.1.3 + github.com/gorilla/handlers v1.4.1 + github.com/gorilla/mux v1.7.3 + github.com/hashicorp/go-immutable-radix v1.1.0 // indirect + github.com/hashicorp/go-msgpack v0.5.5 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/memberlist v0.1.4 github.com/improbable-eng/grpc-web v0.13.0 github.com/kr/text v0.2.0 // indirect github.com/lib/pq v1.8.0 + github.com/miekg/dns v1.1.15 // indirect github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/pelletier/go-toml v1.2.0 + github.com/pelletier/go-toml v1.4.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.1.0 + github.com/prometheus/procfs v0.0.3 // indirect github.com/prometheus/prom2json v1.3.0 github.com/rakyll/statik v0.1.7 - github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/rs/cors v1.7.0 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil/v3 v3.20.11 @@ -54,10 +60,11 @@ require ( golang.org/x/sys v0.0.0-20201214095126-aec9a390925b // indirect golang.org/x/text v0.3.3 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/grpc v1.28.0 + google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 // indirect + google.golang.org/grpc v1.29.1 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.3.0 // indirect - modernc.org/mathutil v1.0.0 + modernc.org/mathutil v1.2.2 modernc.org/strutil v1.0.0 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible ) diff --git a/go.sum b/go.sum index f1ad4ecc5..2b9010aac 100644 --- a/go.sum +++ b/go.sum @@ -16,10 +16,11 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= -github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= -github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/DataDog/datadog-go v2.2.0+incompatible h1:V5BKkxACZLjzHjSgBbr2gvLA2Ae49yhc6CSY7MLy5k4= +github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= +github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -28,8 +29,9 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= @@ -43,6 +45,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -64,10 +68,10 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -86,8 +90,9 @@ github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -97,7 +102,6 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -112,7 +116,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -125,10 +128,10 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= -github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= -github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/handlers v1.4.1 h1:BHvcRGJe/TrL+OqFxoKQGddTgeibiOjaBssV5a/N9sw= +github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -138,23 +141,26 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= +github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc= +github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -174,15 +180,13 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -194,13 +198,14 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= +github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -217,10 +222,12 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -231,6 +238,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -240,27 +248,31 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y= github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= -github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= -github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= @@ -293,17 +305,16 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= @@ -403,7 +414,6 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214095126-aec9a390925b h1:tv7/y4pd+sR8bcNb2D6o7BNU6zjWm0VjQLac+w7fNNM= @@ -417,6 +427,7 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -455,18 +466,19 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU= +google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -485,7 +497,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -498,8 +509,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/http/client_test.go b/http/client_test.go index 87a010501..bb11609d4 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -1462,6 +1462,9 @@ func TestClient_ImportRoaringExists(t *testing.T) { t.Fatal(err) } qr, err := node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + if err != nil { + t.Fatalf(" %v ", err) + } got := qr.Results[0].(*pilosa.Row).Columns() if !reflect.DeepEqual(got, []uint64{}) { t.Fatalf(" Row unexpected columns: got %+v expected: %+v", got, []uint64{}) @@ -1473,6 +1476,9 @@ func TestClient_ImportRoaringExists(t *testing.T) { expected := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537} qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + if err != nil { + t.Fatalf("Query error: %+v", err) + } got = qr.Results[0].(*pilosa.Row).Columns() if !reflect.DeepEqual(got, expected) { t.Fatalf("All unexpected columns: got %+v expected: %+v", got, expected) diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 4773ef666..5794a05d0 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -16,9 +16,10 @@ package pilosa import ( "bytes" - "github.com/pilosa/pilosa/v2/roaring" "reflect" "testing" + + "github.com/pilosa/pilosa/v2/roaring" ) func TestValidateName(t *testing.T) { @@ -73,22 +74,32 @@ func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, e func TestAPI_CombineForExistence(t *testing.T) { bm := roaring.NewBitmap() - bm.Add(pos(1, 1)) - bm.Add(pos(1, 2)) - bm.Add(pos(1, 65537)) //make sure to cross container boundary - bm.Add(pos(1, 65538)) - bm.Add(pos(2, 1)) - bm.Add(pos(2, 2)) - bm.Add(pos(2, 65537)) - bm.Add(pos(2, 65538)) + _, err := bm.Add(pos(1, 1)) + panicOn(err) + _, err = bm.Add(pos(1, 2)) + panicOn(err) + _, err = bm.Add(pos(1, 65537)) //make sure to cross container boundary + panicOn(err) + _, err = bm.Add(pos(1, 65538)) + panicOn(err) + _, err = bm.Add(pos(2, 1)) + panicOn(err) + _, err = bm.Add(pos(2, 2)) + panicOn(err) + _, err = bm.Add(pos(2, 65537)) + panicOn(err) + _, err = bm.Add(pos(2, 65538)) + panicOn(err) buf := new(bytes.Buffer) - _, err := bm.WriteTo(buf) + _, err = bm.WriteTo(buf) panicOn(err) raw := buf.Bytes() - results := combineForExistence(raw) + results, err := combineForExistence(raw) + panicOn(err) bm2 := roaring.NewBitmap() - bm2.ImportRoaringBits(results, false, false, 1< Date: Mon, 22 Feb 2021 08:33:52 -0600 Subject: [PATCH 04/11] update go mod to fix cors bug --- go.mod | 25 +++++++---------- go.sum | 85 +++++++++++++++++++++++++--------------------------------- 2 files changed, 46 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index a9a8ef35d..dcd6b603a 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,7 @@ replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d - github.com/DataDog/datadog-go v2.2.0+incompatible - github.com/OneOfOne/xxhash v1.2.5 // indirect - github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect + github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 github.com/beevik/ntp v0.3.0 github.com/benbjohnson/immutable v0.3.0 github.com/cespare/xxhash v1.1.0 @@ -17,30 +15,26 @@ require ( github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 - github.com/gogo/protobuf v1.3.1 + github.com/gogo/protobuf v1.2.1 github.com/golang/protobuf v1.4.2 github.com/google/go-cmp v0.5.2 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect - github.com/gorilla/handlers v1.4.1 - github.com/gorilla/mux v1.7.3 - github.com/hashicorp/go-immutable-radix v1.1.0 // indirect - github.com/hashicorp/go-msgpack v0.5.5 // indirect - github.com/hashicorp/go-sockaddr v1.0.2 // indirect - github.com/hashicorp/memberlist v0.1.4 + github.com/gorilla/handlers v1.3.0 + github.com/gorilla/mux v1.7.0 + github.com/hashicorp/memberlist v0.1.3 github.com/improbable-eng/grpc-web v0.13.0 github.com/kr/text v0.2.0 // indirect github.com/lib/pq v1.8.0 - github.com/miekg/dns v1.1.15 // indirect github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/pelletier/go-toml v1.4.0 + github.com/pelletier/go-toml v1.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.1.0 - github.com/prometheus/procfs v0.0.3 // indirect github.com/prometheus/prom2json v1.3.0 github.com/rakyll/statik v0.1.7 + github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/rs/cors v1.7.0 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil/v3 v3.20.11 @@ -60,11 +54,10 @@ require ( golang.org/x/sys v0.0.0-20201214095126-aec9a390925b // indirect golang.org/x/text v0.3.3 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 // indirect - google.golang.org/grpc v1.29.1 + google.golang.org/grpc v1.28.0 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.3.0 // indirect - modernc.org/mathutil v1.2.2 + modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible ) diff --git a/go.sum b/go.sum index 2b9010aac..f1ad4ecc5 100644 --- a/go.sum +++ b/go.sum @@ -16,11 +16,10 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= -github.com/DataDog/datadog-go v2.2.0+incompatible h1:V5BKkxACZLjzHjSgBbr2gvLA2Ae49yhc6CSY7MLy5k4= -github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= +github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= -github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -29,9 +28,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= @@ -45,8 +43,6 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -68,10 +64,10 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -90,9 +86,8 @@ github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -102,6 +97,7 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -116,6 +112,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -128,10 +125,10 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/handlers v1.4.1 h1:BHvcRGJe/TrL+OqFxoKQGddTgeibiOjaBssV5a/N9sw= -github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= +github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= +github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -141,26 +138,23 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc= -github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= -github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= -github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -180,13 +174,15 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -198,14 +194,13 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= -github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -222,12 +217,10 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= -github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -238,7 +231,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -248,31 +240,27 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y= github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= @@ -305,16 +293,17 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= @@ -414,6 +403,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214095126-aec9a390925b h1:tv7/y4pd+sR8bcNb2D6o7BNU6zjWm0VjQLac+w7fNNM= @@ -427,7 +417,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -466,19 +455,18 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU= -google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -497,6 +485,7 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -509,8 +498,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY= -modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= From 155a4b4a316ccb0baf8d15321b668286dc7d66bf Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 22 Feb 2021 10:07:32 -0600 Subject: [PATCH 05/11] comment clarification --- api.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index f67ecf100..34976f0df 100644 --- a/api.go +++ b/api.go @@ -423,9 +423,10 @@ func importWorker(importWork chan importJob) { case RequestActionSet: fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) data := viewData - if fileMagic != roaring.MagicNumber { // if pilosa roaring format - // must make a copy of data to operate on locally on standard roaring format. - // field.importRoaring changes the standard roaring run format to pilosa roaring + if fileMagic != roaring.MagicNumber { + // if the view data arrives is in the "standard" roaring format, we must + // make a copy of data in order allow for the convertion to the pilosa roaring run format + // in field.importRoaring data = make([]byte, len(viewData)) copy(data, viewData) } From 1b87b7148ee7e8937cd3e71e3b97a2fe44cdc3ed Mon Sep 17 00:00:00 2001 From: tgruben Date: Mon, 22 Feb 2021 11:06:55 -0600 Subject: [PATCH 06/11] Update api.go Co-authored-by: Matthew Jaffee --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 34976f0df..7c3bc8ceb 100644 --- a/api.go +++ b/api.go @@ -425,7 +425,7 @@ func importWorker(importWork chan importJob) { data := viewData if fileMagic != roaring.MagicNumber { // if the view data arrives is in the "standard" roaring format, we must - // make a copy of data in order allow for the convertion to the pilosa roaring run format + // make a copy of data in order allow for the conversion to the pilosa roaring run format // in field.importRoaring data = make([]byte, len(viewData)) copy(data, viewData) From c2da1d267158171338acc74a47a48b0b9195429c Mon Sep 17 00:00:00 2001 From: tgruben Date: Mon, 22 Feb 2021 11:07:41 -0600 Subject: [PATCH 07/11] Update api.go Co-authored-by: Matthew Jaffee --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 7c3bc8ceb..3a7dc36cf 100644 --- a/api.go +++ b/api.go @@ -465,7 +465,7 @@ func importWorker(importWork chan importJob) { } } -// merge all rows to singled existence row +// combineForExistence unions all rows in the fragment to be imported into a single row to update the existence field. TODO: It would probably be more efficient to only unmarshal the input data once, and use the calculated existence Bitmap directly rather than returning it to bytes, but most of our ingest paths update existence separately, so it's more important that this just be obviously correct at the moment. func combineForExistence(inputRoaringData []byte) ([]byte, error) { rowSize := uint64(1 << shardVsContainerExponent) rit, err := roaring.NewRoaringIterator(inputRoaringData) From 5cbbb7996e28b09054fa55426c4b5fc582701405 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 22 Feb 2021 11:37:30 -0600 Subject: [PATCH 08/11] jaffee test handling suggestions --- pilosa_internal_test.go | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 5794a05d0..693dc52cb 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -73,30 +73,15 @@ func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, e } func TestAPI_CombineForExistence(t *testing.T) { - bm := roaring.NewBitmap() - _, err := bm.Add(pos(1, 1)) - panicOn(err) - _, err = bm.Add(pos(1, 2)) - panicOn(err) - _, err = bm.Add(pos(1, 65537)) //make sure to cross container boundary - panicOn(err) - _, err = bm.Add(pos(1, 65538)) - panicOn(err) - _, err = bm.Add(pos(2, 1)) - panicOn(err) - _, err = bm.Add(pos(2, 2)) - panicOn(err) - _, err = bm.Add(pos(2, 65537)) - panicOn(err) - _, err = bm.Add(pos(2, 65538)) - panicOn(err) - + bm := roaring.NewBitmap(pos(1, 1), pos(1, 2), pos(1, 65537), pos(1, 65538), pos(2, 1), pos(2, 2), pos(2, 65537), pos(2, 65538)) buf := new(bytes.Buffer) - _, err = bm.WriteTo(buf) + _, err := bm.WriteTo(buf) panicOn(err) raw := buf.Bytes() results, err := combineForExistence(raw) - panicOn(err) + if err != nil { + t.Fatalf("failure to combine: %v", err) + } bm2 := roaring.NewBitmap() _, _, err = bm2.ImportRoaringBits(results, false, false, 1< Date: Mon, 22 Feb 2021 11:42:48 -0600 Subject: [PATCH 09/11] Update roaring/roaring.go Co-authored-by: Matthew Jaffee --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 37adfc2d5..39bdbc714 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2282,7 +2282,7 @@ func (b *Bitmap) ImportRoaringBits(data []byte, clear bool, log bool, rowSize ui func (b *Bitmap) MergeRoaringRawIteratorIntoExists(itr RoaringIterator, rowSize uint64) error { if itr == nil { - return errors.New("bad roaring iterator, but don't know why") + return errors.New("nil RoaringIterator passed to MergeRoaringRawIteratorIntoExists") } var synthC Container importUpdater := func(oldC *Container, existed bool) (newC *Container, write bool) { From d220426686d51a473b0b999c56737b9f46e5e557 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 22 Feb 2021 11:50:01 -0600 Subject: [PATCH 10/11] doc comment --- roaring/roaring.go | 80 ++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 39bdbc714..1ee979bde 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2279,46 +2279,50 @@ func (b *Bitmap) ImportRoaringBits(data []byte, clear bool, log bool, rowSize ui } return b.ImportRoaringRawIterator(itr, clear, log, rowSize) } + +// MergeRoaringRawIteratorIntoExists is a special merge iterator that flattens the results onto a +// single row which is used for determining existence. All row references are removed and only the column +// is considered. func (b *Bitmap) MergeRoaringRawIteratorIntoExists(itr RoaringIterator, rowSize uint64) error { - if itr == nil { - return errors.New("nil RoaringIterator passed to MergeRoaringRawIteratorIntoExists") - } - var synthC Container - importUpdater := func(oldC *Container, existed bool) (newC *Container, write bool) { - existN := oldC.N() - if existN == MaxContainerVal+1 { - return oldC, false - } - if existN == 0 { - newerC := synthC.Clone() - return newerC, true - } - newC = oldC.unionInPlace(&synthC) - if newC.typeID == ContainerBitmap { - newC.Repair() - } - if newC.N() != existN { - return newC, true - } - return oldC, false - } - itrKey, itrCType, itrN, itrLen, itrPointer, itrErr := itr.Next() - for itrErr == nil { - synthC.typeID = itrCType - synthC.n = int32(itrN) - synthC.len = int32(itrLen) - synthC.cap = int32(itrLen) - synthC.pointer = itrPointer - b.Containers.Update(itrKey%rowSize, importUpdater) - itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next() - } - // note: if we get a non-EOF err, it's possible that we made SOME - // changes but didn't log them. I don't have a good solution to this. - if itrErr != io.EOF { - return itrErr - } - return nil + if itr == nil { + return errors.New("nil RoaringIterator passed to MergeRoaringRawIteratorIntoExists") + } + var synthC Container + importUpdater := func(oldC *Container, existed bool) (newC *Container, write bool) { + existN := oldC.N() + if existN == MaxContainerVal+1 { + return oldC, false + } + if existN == 0 { + newerC := synthC.Clone() + return newerC, true + } + newC = oldC.unionInPlace(&synthC) + if newC.typeID == ContainerBitmap { + newC.Repair() + } + if newC.N() != existN { + return newC, true + } + return oldC, false + } + itrKey, itrCType, itrN, itrLen, itrPointer, itrErr := itr.Next() + for itrErr == nil { + synthC.typeID = itrCType + synthC.n = int32(itrN) + synthC.len = int32(itrLen) + synthC.cap = int32(itrLen) + synthC.pointer = itrPointer + b.Containers.Update(itrKey%rowSize, importUpdater) + itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next() + } + // note: if we get a non-EOF err, it's possible that we made SOME + // changes but didn't log them. I don't have a good solution to this. + if itrErr != io.EOF { + return itrErr + } + return nil } func (b *Bitmap) ImportRoaringRawIterator(itr RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { From 081d86c04b1e128789c7ea0c06fca765f54b4e85 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 22 Feb 2021 15:06:45 -0600 Subject: [PATCH 11/11] mixed row test --- pilosa_internal_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 693dc52cb..b984d5680 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -73,7 +73,7 @@ func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, e } func TestAPI_CombineForExistence(t *testing.T) { - bm := roaring.NewBitmap(pos(1, 1), pos(1, 2), pos(1, 65537), pos(1, 65538), pos(2, 1), pos(2, 2), pos(2, 65537), pos(2, 65538)) + bm := roaring.NewBitmap(pos(1, 1), pos(1, 2), pos(1, 3), pos(1, 65537), pos(1, 65538), pos(2, 1), pos(2, 2), pos(2, 5), pos(2, 65537), pos(2, 65538)) buf := new(bytes.Buffer) _, err := bm.WriteTo(buf) panicOn(err) @@ -85,7 +85,7 @@ func TestAPI_CombineForExistence(t *testing.T) { bm2 := roaring.NewBitmap() _, _, err = bm2.ImportRoaringBits(results, false, false, 1<