From 5b43a84a3be25519cc2ce0fb2f68c6f46552aa42 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Feb 2021 17:09:50 -0600 Subject: [PATCH 01/28] . --- cmd/sauron/design.org | 199 ++++ cmd/sauron/httputil.go | 274 +++++ cmd/sauron/main.go | 1802 +++++++++++++++++++++++++++++++++ cmd/sauron/populate.sh | 10 + cmd/sauron/sample_schema.json | 1 + cmd/sauron/sauron_test.go | 396 ++++++++ cmd/sauron/schema_test.go | 353 +++++++ cmd/sauron/sqlgen_test.go | 260 +++++ cmd/sauron/test_sqlite.sh | 51 + cmd/sauron/vprint.go | 167 +++ 10 files changed, 3513 insertions(+) create mode 100644 cmd/sauron/design.org create mode 100644 cmd/sauron/httputil.go create mode 100644 cmd/sauron/main.go create mode 100644 cmd/sauron/populate.sh create mode 100644 cmd/sauron/sample_schema.json create mode 100644 cmd/sauron/sauron_test.go create mode 100644 cmd/sauron/schema_test.go create mode 100644 cmd/sauron/sqlgen_test.go create mode 100755 cmd/sauron/test_sqlite.sh create mode 100644 cmd/sauron/vprint.go diff --git a/cmd/sauron/design.org b/cmd/sauron/design.org new file mode 100644 index 000000000..aa579ab34 --- /dev/null +++ b/cmd/sauron/design.org @@ -0,0 +1,199 @@ + +* Oracle Design +- setup server that intersepts PQL queries and proxies to live pilosa server +- implement all the pilosa functions using SQL, specifically the sqlite flavor. This will + provide validation against pilosa results +- provide timings for each query path, PQL and SQL and compare +- this is intended to be a correctness check and not a performance check. the + sql solution is likely to be pretty slow with realitively small numbers ~1millions + #+BEGIN_SRC + .─────. + ,' `. ┌────────────┐ ┌────────┐ +;pilosa json: │ Sauron │ ┌──▶│ pilosa │ +: client ;──────────▶ │ │───┤ └────────┘ + ╲ ╱ │ │ │ ┌────────┐ + `. ,' └────────────┘ └──▶│ sqlite │ + `───' └────────┘ + #+END_SRC +* Setup Sql (sqlite) +#+name: setup test +#+header: :results silent +#+header: :db poc.db + #+BEGIN_SRC sqlite + create table bits (field string,row string, column bigint); + create table bsi (field string,column bigint , val bigint); + CREATE VIEW columns as + select distinct column from bits + UNION + select distinct column from bsi; + + INSERT INTO bits VALUES( 'color','red',1); + INSERT INTO bits VALUES( 'color','red',2); + INSERT INTO bits VALUES( 'color','red',3); + INSERT INTO bits VALUES( 'color','red',4); + INSERT INTO bits VALUES( 'color','red',5); + INSERT INTO bits VALUES( 'color','green',2); + INSERT INTO bits VALUES( 'color','green',4); + INSERT INTO bits VALUES( 'color','green',6); + INSERT INTO bits VALUES( 'color','green',8); + INSERT INTO bits VALUES( 'color','yellow',1); + INSERT INTO bits VALUES( 'color','yellow',3); + INSERT INTO bits VALUES( 'color','yellow',5); + INSERT INTO bits VALUES( 'color','yellow',7); + INSERT INTO bits VALUES( 'color','orange',5); + INSERT INTO bits VALUES( 'color','orange',6); + INSERT INTO bits VALUES( 'color','orange',7); + INSERT INTO bits VALUES( 'color','orange',8); + + INSERT INTO bsi VALUES( 'luminosity',1, 1); + INSERT INTO bsi VALUES( 'luminosity',2, 2); + INSERT INTO bsi VALUES( 'luminosity',3, 4); + INSERT INTO bsi VALUES( 'luminosity',4, 5); + INSERT INTO bsi VALUES( 'luminosity',5, 4); + INSERT INTO bsi VALUES( 'luminosity',6, 3); + INSERT INTO bsi VALUES( 'luminosity',7, 2); + INSERT INTO bsi VALUES( 'luminosity',8, 1); + #+END_SRC + +* TODO PQL to SQL TODO [10/11] +- [X] Row(color=red) + #+begin_src go +(*pql.Query)(&pql.Query{ + Calls: ([]*pql.Call)([]*pql.Call{ + (*pql.Call)(&pql.Call{ + Name: (string)("Row"), + Args: (map[string]interface{})(map[string]interface{}{ + (string)("color"): (string)("red"), + }), + Children: ([]*pql.Call)(nil), + Type: (pql.CallType)(0), + Precomputed: (map[uint64]interface{})(nil), + }), + }), + callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), + conditional: ([]string)(nil), +}) + #+end_src + #+BEGIN_SRC sqlite + select column from bits where field="color" AND row="red" + #+END_SRC +- [X] Count(Row(color=red)) + #+begin_src go +(*pql.Query)(&pql.Query{ + Calls: ([]*pql.Call)([]*pql.Call{ + (*pql.Call)(&pql.Call{ + Name: (string)("Count"), + Args: (map[string]interface{})(nil), + Children: ([]*pql.Call)([]*pql.Call{ + (*pql.Call)(&pql.Call{ + Name: (string)("Row"), + Args: (map[string]interface{})(map[string]interface{}{ + (string)("color"): (string)("red"), + }), + Children: ([]*pql.Call)(nil), + Type: (pql.CallType)(0), + Precomputed: (map[uint64]interface{})(nil), + }), + }), + Type: (pql.CallType)(0), + Precomputed: (map[uint64]interface{})(nil), + }), + }), + callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), + conditional: ([]string)(nil), +}) + + #+end_src + #+BEGIN_SRC sql + select count(*) from( + select column from bits where field="color" AND row="red" + ) + #+END_SRC +- [X] Intersect(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from bits where field="color" AND row="red" + intersect + select column from bits where field="color" AND row="green" + #+END_SRC +- [X] Union(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from bits where field="color" AND row="red" + union + select column from bits where field="color" AND row="green" + #+END_SRC +- [X] Difference(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from bits where field="color" AND row="red" + except + select column from bits where field="color" AND row="green" + #+END_SRC +- [X] Not(Row(color=red)) + #+BEGIN_SRC sql + select column from columns + except + select column from bits where field="color" AND row="red" + #+END_SRC +- [X] Xor(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from ( + select column from bits where field="color" AND row="green" + union + select column from bits where field="color" AND row="red" + ) + except + select column from ( + select column from bits where field="color" AND row="green" + intersect + select column from bits where field="color" AND row="red" + ) + #+END_SRC +- [X] Distinct(field=luminosity) + #+BEGIN_SRC sql + select distinct val from bsi where field="lumin" + #+END_SRC +- [X] Distinct(Row(color="red"),field=luminosity) + #+BEGIN_SRC sql + select distinct val from bsi + where field="luminosity" ANDcolumn in ( + select column from bits where row="color" AND row="red" + ) + #+END_SRC +- [X] Distinct(Intersect(Row(color="red"),Row(color="green")),luminosity) + #+BEGIN_SRC sql + select distinct val from bsi + where column in ( + select column from bits where field="color" AND row="red" + intersect + select column from bits where field="color" AND row="green" + ) + AND field="luminosity" + #+END_SRC + +- [X] Count(Union(Intersect(Row(color=red),Row(color=green)),Intersect(Row(color=yellow),Row(color=orange)))) +#+begin_src sqlite +select count(*) from ( +select column from ( + select column from bits where field="color" AND row="green" + intersect + select column from bits where field="color" AND row="red" +) +union +select column from ( + select column from bits where field="color" AND row="yellow" + intersect + select column from bits where field="color" AND row="orange" +)) + +- [ ] Rows(color) +#+begin_src sqlite +select distinct row from bits where field="color" +#+end_src + +- [ ] GroupBy(Rows(color)) + #+BEGIN_SRC sql + select row,count(*) from bits where field="color" group by row; + #+END_SRC +- [ ] GroupBy(Rows(color), Rows(other), limit=7) + #+BEGIN_SRC sql + select row,count(*) from bits where field="color" group by row; + #+END_SRC diff --git a/cmd/sauron/httputil.go b/cmd/sauron/httputil.go new file mode 100644 index 000000000..7056d40e8 --- /dev/null +++ b/cmd/sauron/httputil.go @@ -0,0 +1,274 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "log" + "mime" + "net/http" + "strings" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/pql" + "github.com/pkg/errors" +) + +// validHeaderAcceptJSON returns false if one or more Accept +// headers are present, but none of them are "application/json" +// (or any matching wildcard). Otherwise returns true. +func validHeaderAcceptJSON(header http.Header) bool { + return validHeaderAcceptType(header, "application", "json") +} + +func validHeaderAcceptType(header http.Header, typ, subtyp string) bool { + if v, found := header["Accept"]; found { + for _, v := range v { + t, _, err := mime.ParseMediaType(v) + if err != nil { + switch err { + case mime.ErrInvalidMediaParameter: + // This is an optional feature, so we can keep going anyway. + default: + continue + } + } + spl := strings.SplitN(t, "/", 2) + if len(spl) < 2 { + continue + } + switch { + case spl[0] == typ && spl[1] == subtyp: + return true + case spl[0] == "*" && spl[1] == subtyp: + return true + case spl[0] == typ && spl[1] == "*": + return true + case spl[0] == "*" && spl[1] == "*": + return true + } + } + return false + } + return true +} + +// successResponse is a general success/error struct for http responses. +type successResponse struct { + //h *Handler + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + //Error *Error `json:"error,omitempty"` + Error error +} + +// check determines success or failure based on the error. +// It also returns the corresponding http status code. +func (r *successResponse) check(err error) (statusCode int) { + if err == nil { + r.Success = true + return 0 + } + + cause := errors.Cause(err) + + // Determine HTTP status code based on the error type. + switch cause.(type) { + case pilosa.BadRequestError: + statusCode = http.StatusBadRequest + case pilosa.ConflictError: + statusCode = http.StatusConflict + case pilosa.NotFoundError: + statusCode = http.StatusNotFound + default: + statusCode = http.StatusInternalServerError + } + + r.Success = false + r.Error = err // = &Error{Message: err.Error()} + + return statusCode +} + +// write sends a response to the http.ResponseWriter based on the success +// status and the error. +func (r *successResponse) write(w http.ResponseWriter, err error) { + // Apply the error and get the status code. + statusCode := r.check(err) + + // Marshal the json response. + msg, err := json.Marshal(r) + if err != nil { + http.Error(w, string(msg), http.StatusInternalServerError) + return + } + + // Write the response. + if statusCode == 0 { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write(msg) + if err != nil { + log.Printf("error writing response: %v", err) + return + } + _, err = w.Write([]byte("\n")) + if err != nil { + log.Printf("error writing newline after response: %v", err) + return + } + } else { + http.Error(w, string(msg), statusCode) + } +} + +type postFieldRequest struct { + Options fieldOptions `json:"options"` +} + +// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values, +// and used for input validation. +type fieldOptions struct { + Type string `json:"type,omitempty"` + CacheType *string `json:"cacheType,omitempty"` + CacheSize *uint32 `json:"cacheSize,omitempty"` + Min *pql.Decimal `json:"min,omitempty"` + Max *pql.Decimal `json:"max,omitempty"` + Scale *int64 `json:"scale,omitempty"` + TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` + Keys *bool `json:"keys,omitempty"` + NoStandardView bool `json:"noStandardView,omitempty"` + ForeignIndex *string `json:"foreignIndex,omitempty"` +} + +func (o *fieldOptions) ToPilosaFieldOptions() *pilosa.FieldOptions { + + fo := &pilosa.FieldOptions{ + Type: o.Type, + + /* + //Base int64 `json:"base,omitempty"` + //BitDepth uint `json:"bitDepth,omitempty"` + Min: *o.Min, + Max: *o.Max, + Scale: *o.Scale, + NoStandardView: o.NoStandardView, + CacheSize: *o.CacheSize, + CacheType: *o.CacheType, + TimeQuantum: *o.TimeQuantum, + ForeignIndex: *o.ForeignIndex, + */ + } + if o.Keys != nil { + fo.Keys = *o.Keys + } + return fo +} + +func (o *fieldOptions) validate() error { + // Pointers to default values. + defaultCacheType := pilosa.DefaultCacheType + defaultCacheSize := uint32(pilosa.DefaultCacheSize) + + switch o.Type { + case pilosa.FieldTypeSet, "": + // Because FieldTypeSet is the default, its arguments are + // not required. Instead, the defaults are applied whenever + // a value does not exist. + if o.Type == "" { + o.Type = pilosa.FieldTypeSet + } + if o.CacheType == nil { + o.CacheType = &defaultCacheType + } + if o.CacheSize == nil { + o.CacheSize = &defaultCacheSize + } + if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type set")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type set")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) + } + case pilosa.FieldTypeInt: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { + return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) + } + case pilosa.FieldTypeDecimal: + if o.Scale == nil { + return pilosa.NewBadRequestError(errors.New("decimal field requires a scale argument")) + } else if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { + return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) + } + case pilosa.FieldTypeTime: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time")) + } else if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type time")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type time")) + } else if o.TimeQuantum == nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time")) + } + case pilosa.FieldTypeMutex: + if o.CacheType == nil { + o.CacheType = &defaultCacheType + } + if o.CacheSize == nil { + o.CacheSize = &defaultCacheSize + } + if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type mutex")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) + } + case pilosa.FieldTypeBool: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) + } else if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) + } else if o.Keys != nil { + return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool")) + } else if o.ForeignIndex != nil { + return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key")) + } + default: + return errors.Errorf("invalid field type: %s", o.Type) + } + return nil +} diff --git a/cmd/sauron/main.go b/cmd/sauron/main.go new file mode 100644 index 000000000..497e5346b --- /dev/null +++ b/cmd/sauron/main.go @@ -0,0 +1,1802 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + //"io" + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "os" + "os/exec" + "sort" + "strconv" + "strings" + "time" + + "modernc.org/sqlite" + _ "modernc.org/sqlite" + + //"github.com/gorilla/handlers" + "github.com/gorilla/mux" + "github.com/pilosa/pilosa/v2" + + "github.com/pilosa/pilosa/v2/encoding/proto" + "github.com/pilosa/pilosa/v2/pql" + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pkg/errors" + "github.com/shurcooL/go-goon" +) + +func main() { + +} + +// Sauron sees everything. +// And he converts PQL into SQL. +// +// The PQL is applied to a Pilosa and the SQL +// is applied to a SQL database. Then the +// query results are compared as they are +// returned to the client. +// +type Sauron struct { + cfg *SauronConfig + + sql *PQLToSQL + url string // "https://..." + addr string // host:port + mux *http.ServeMux + srv *http.Server + cmd *exec.Cmd + + // target of proxy + pilosaHttpURL string + + Handler http.Handler +} + +type SauronConfig struct { + Bind string // host:port + BindGRPC string // host:port + GossipPort string // port + DataDir string + + DatabaseName string // for sql database +} + +// IndexInfo2 and FieldInfo2 let us look up +// the schema details quickly by using maps +// to index the fields in an index. +type IndexInfo2 struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options pilosa.IndexOptions `json:"options"` + Fields map[string]*FieldInfo2 `json:"fields"` + ShardWidth uint64 `json:"shardWidth"` +} + +type FieldInfo2 struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options pilosa.FieldOptions `json:"options"` + Views []*pilosa.ViewInfo `json:"views,omitempty"` + indexInfo *IndexInfo2 +} + +func (fi *FieldInfo2) GetIndexName() string { + return fi.indexInfo.Name +} + +func (fi *FieldInfo2) GetIndexCreatedAt() int64 { + return fi.indexInfo.CreatedAt +} + +func (fi *FieldInfo2) GetCreatedAt() int64 { + return fi.CreatedAt +} + +func (fi *FieldInfo2) GetName() string { + return fi.Name +} + +type PQLToSQL struct { + schema pilosa.Schema + DB *sql.DB + + dbprefix string // prefixed to all tables + + // map from indexName to *IndexInfo2 + i2f map[string]*IndexInfo2 + ShardWidth uint64 +} + +func NewPQLToSQL(dbprefix string) *PQLToSQL { + return &PQLToSQL{ + dbprefix: strings.ToLower(dbprefix), // some SQL are not case sensitive, just use lower case. + i2f: make(map[string]*IndexInfo2), + ShardWidth: 1048576, + } +} + +func (ps *PQLToSQL) Start() error { + db, err := sql.Open("sqlite", fmt.Sprintf("%v.db", ps.dbprefix)) + if err != nil { + log.Fatal(err) + } + ps.DB = db + return nil +} +func (ps *PQLToSQL) MustRemove() { + name := fmt.Sprintf("%v.db", ps.dbprefix) + os.Remove(name) + // panicOn(err) +} + +func (ps *PQLToSQL) Stop() error { + return ps.DB.Close() +} +func (ps *PQLToSQL) Store(index, field string, rowIdOrKey interface{}, rowCall []*pql.Call) (bool, error) { + rowSql, _, _, err := ToSql(rowCall, index) + panicOn(err) + insertSql := fmt.Sprintf(`insert into %vλbits select "%v" as field, "%v" as row, column,0 as timestamp from ( %v )`, index, field, rowIdOrKey, rowSql) + res, err := ps.DB.Exec(insertSql) + if err != nil { + return false, err + } + aff, err := res.RowsAffected() + if err != nil { + return false, err + } + return aff > 0, nil +} + +// convert '-' to 'Θ' since db cannot have '-' in table names, +// but pilosa allows it in index and field names. +func dash2theta(s string) (r string) { + return strings.Replace(s, "-", "Θ", -1) +} + +func theta2dash(s string) (r string) { + return strings.Replace(s, "Θ", "-", -1) +} +func (ps *PQLToSQL) CreateSchema(jsonBytes []byte) (err error) { + err = json.Unmarshal(jsonBytes, &ps.schema) + if err != nil { + panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) + } + for _, i := range ps.schema.Indexes { + ps.createDatabase(i.Name) + for _, f := range i.Fields { + panicOn(ps.CreateField(i, f)) + } + } + return nil +} + +func (ps *PQLToSQL) CreateFieldWithOptions(indexName, fieldName string, o fieldOptions) error { + + ii2, ok := ps.i2f[indexName] + if !ok { + return fmt.Errorf("index '%v' not found", indexName) + } + + f2, ok := ii2.Fields[fieldName] + if !ok { + f2 = &FieldInfo2{ + Name: fieldName, + //CreatedAt: f.CreatedAt, + Options: *(o.ToPilosaFieldOptions()), + //Views: f.Views, + indexInfo: ii2, + } + ii2.Fields[fieldName] = f2 + } + return nil +} + +func (ps *PQLToSQL) CreateIndex(indexName string, opts *pilosa.IndexOptions) (err error) { + ii2, ok := ps.i2f[indexName] + if !ok { + ii2 = &IndexInfo2{ + Name: indexName, + //CreatedAt: i.CreatedAt, + Options: *opts, + Fields: make(map[string]*FieldInfo2), + ShardWidth: 1048576, + } + //TODO (twg) figure oiut where to get Shardwidht + ps.i2f[indexName] = ii2 + } + return nil +} +func (ps *PQLToSQL) Query(index string, query []*pql.Call) (err error) { + // sql, err := ToSQL(query, index) + //TODO (twg) implement QUery + return nil +} +func (ps *PQLToSQL) CreateField(i *pilosa.IndexInfo, f *pilosa.FieldInfo) (err error) { + ii2, ok := ps.i2f[i.Name] + if !ok { + ii2 = &IndexInfo2{ + Name: i.Name, + CreatedAt: i.CreatedAt, + Options: i.Options, + Fields: make(map[string]*FieldInfo2), + ShardWidth: i.ShardWidth, + } + ps.i2f[i.Name] = ii2 + } + + f2, ok := ii2.Fields[f.Name] + if !ok { + f2 = &FieldInfo2{ + Name: f.Name, + CreatedAt: f.CreatedAt, + Options: f.Options, + Views: f.Views, + indexInfo: ii2, + } + ii2.Fields[f.Name] = f2 + } + return nil +} + +func (ps *PQLToSQL) createDatabase(index string) (err error) { + idx_prefix := dash2theta(index) + "λ" + // Create fields that don't exist. + sql := fmt.Sprintf("create table %vbits (field string,row,column,timestamp, unique(field,row,column,timestamp) )", idx_prefix) + _, err = ps.DB.Exec(sql) + panicOn(err) + sql = fmt.Sprintf("create table %vbsi (field,column, val bigint,unique(field,column)) ", idx_prefix) + _, err = ps.DB.Exec(sql) + panicOn(err) + sql = fmt.Sprintf(`CREATE VIEW %vcolumns as + select distinct column from %vbits + UNION + select distinct column from %vbsi`, idx_prefix, idx_prefix, idx_prefix) + _, err = ps.DB.Exec(sql) + panicOn(err) + return +} +func (ps *PQLToSQL) GetField(indexName, fieldName string) (*FieldInfo2, error) { + idx, ok := ps.i2f[indexName] + if !ok { + return nil, errors.New(fmt.Sprintf("index not found '%v'", indexName)) + } + f, ok := idx.Fields[fieldName] + if !ok { + return nil, errors.New(fmt.Sprintf("field not found '%v' in index '%v'", fieldName, indexName)) + } + return f, nil +} + +//TODO (twg) below will be moved to its own file later just have it here for quick navigation +type PqlFunction uint8 + +//debate between this +type ResultType uint8 + +const ( + UNKNOWN = ResultType(iota) + BITMAP + INT + ARRAY + GROUPCOUNT + SIGNEDROW + BOOL + PAIRSFIELD + TABLE + AGGSQL +) + +func getAggregateSql(call *pql.Call, index string) (string, error) { + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", errors.New("no _field present") + } + } + var sq string + if len(call.Children) > 0 { + //only child should be a bitmap call + var err error + sq, _, _, err = ToSql(call.Children, index) + if err != nil { + return sq, err + } + sq = " and column in (" + sq + ")" + + } + sqb := fmt.Sprintf(`select %v(val) from %vλbsi where field="%v"%v`, strings.ToLower(call.Name), index, fieldName, sq) + sql := fmt.Sprintf(`select val,count(*) from %vλbsi where val=(%v)%v`, index, sqb, sq) + + return sql, nil +} + +// ToSql takes a pilosa query(ast) and converts it to sql +// assumption: the pql has successfully executed on server +func applyCondition(field string, cond *pql.Condition) (sql string) { + + switch cond.Op { + case pql.EQ, pql.NEQ, pql.LT, pql.LTE, pql.GT, pql.GTE: + val, ok := cond.Uint64Value() + if !ok { + panic("no value for condition") + } + if cond.Op == pql.EQ { + sql = fmt.Sprintf(`field="%v" and val=%d`, field, val) + } else if cond.Op == pql.NEQ { + sql = fmt.Sprintf(`field="%v" and val!=%d`, field, val) + } else if cond.Op == pql.LT { + sql = fmt.Sprintf(`field="%v" and val<%d`, field, val) + } else if cond.Op == pql.LTE { + sql = fmt.Sprintf(`field="%v" and val<=%d`, field, val) + } else if cond.Op == pql.GT { + sql = fmt.Sprintf(`field="%v" and val>%d`, field, val) + } else if cond.Op == pql.GTE { + sql = fmt.Sprintf(`field="%v" and val>=%d`, field, val) + } + case pql.BETWEEN, pql.BTWN_LT_LTE, pql.BTWN_LTE_LT, pql.BTWN_LT_LT: + val, ok := cond.Uint64SliceValue() + if !ok { + panic("bad value for condition") + } + if cond.Op == pql.BETWEEN { + sql = fmt.Sprintf(`field="%v" and %v>=%d and %v<=%d`, field, "val", val[0], "val", val[1]) + } else if cond.Op == pql.BTWN_LT_LTE { + sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[1]) + } else if cond.Op == pql.BTWN_LTE_LT { + sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[0]) + } else if cond.Op == pql.BTWN_LT_LT { + sql = fmt.Sprintf(`field="%v" and %v>%d and %v<%d`, field, "val", val[0], "val", val[1]) + } + } + return +} +func FetchFields(rowsCalls []*pql.Call) ([]string, []interface{}, error) { + //TODO (twg) date fields in rows? + results := make([]string, len(rowsCalls)) + columns := make([]interface{}, 0) + for i, call := range rowsCalls { + //TODO (twg) Time Fields ie from/to args + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return nil, nil, errors.New("no field for rows") + } + } + results[i] = fieldName.(string) + col, ok := call.Args["column"] + if ok { + s := fmt.Sprintf("f%v.column='%v'", i+1, col) + columns = append(columns, s) + } + } + return results, columns, nil +} + +func ToSql(ast []*pql.Call, index string) (string, ResultType, []interface{}, error) { + resultType := UNKNOWN + optional := make([]interface{}, 0) + if len(ast) == 0 { + return "", resultType, optional, nil + } + stack := make([]string, 0) + for _, call := range ast { + switch c := strings.ToLower(call.Name); c { + case "row": + var sql string + if call.HasConditionArg() { + for k, v := range call.Args { + csql := applyCondition(k, v.(*pql.Condition)) + sql = fmt.Sprintf("select distinct column from %vλbsi where %v", index, csql) + + break + } + } else { + field, err := call.FieldArg() + panicOn(err) + row := call.Args[field] + + t, ok := call.Args["from"] + var timestampClause string + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause = fmt.Sprintf(` AND timestamp >= "%v"`, clean) + } + t, ok = call.Args["to"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause += fmt.Sprintf(` AND timestamp < "%v"`, clean) + + } + + sql = fmt.Sprintf(`select distinct column from %vλbits where field="%v" AND row="%v"%v`, index, field, row, timestampClause) + + } + stack = append(stack, sql) + resultType = BITMAP + case "count": + //should have args and should be top level + if len(call.Children) == 0 { + return "", resultType, optional, errors.New("invalid args for count") + } + nestedSQL, _, _, err := ToSql(call.Children, index) + panicOn(err) + sql := "select count(*) from( " + nestedSQL + " )" + stack = append(stack, sql) + resultType = INT + case "union", "intersect": + var sql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(sql) == 0 { + sql = sq + } else { + sql += " " + c + " " + sq + } + } + sql = "select column from(" + sql + ")" + stack = append(stack, sql) + resultType = BITMAP + case "difference": + var sql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(sql) == 0 { + sql = sq + } else { + sql += " except " + sq + } + } + sql = "select column from(" + sql + ")" + stack = append(stack, sql) + resultType = BITMAP + case "not": + //should have args and should be top level + if len(call.Children) == 0 { + return "", resultType, optional, errors.New("invalid args for count") + } + nestedSQL, _, _, err := ToSql(call.Children, index) + panicOn(err) + sql := fmt.Sprintf("select column from %vλcolumns except %v", index, nestedSQL) + stack = append(stack, sql) + resultType = BITMAP + case "xor": + var unionSql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(unionSql) == 0 { + unionSql = sq + } else { + unionSql += " union " + sq + } + } + unionSql = "select column from(" + unionSql + ")" + var intersectSql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(intersectSql) == 0 { + intersectSql = sq + } else { + intersectSql += " intersect " + sq + } + } + intersectSql = "select column from(" + intersectSql + ")" + sql := unionSql + " except " + intersectSql + stack = append(stack, sql) + resultType = BITMAP + + case "distinct": + //TODO validate field type, only valid for bsi fields + fieldName, ok := call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("field not specified on distinct call") + } + var sql string + if len(call.Children) > 1 { + return "", resultType, optional, errors.New(fmt.Sprintf("unexpected args to distinct query:'%v'", call.Children)) + } + sql = fmt.Sprintf(`select distinct val from %vλbsi where field="%v"`, index, fieldName) + if len(call.Children) > 0 { + sq, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) + panicOn(err) + sql += " AND column in (" + sq + ")" + } + stack = append(stack, sql) + resultType = SIGNEDROW + case "rows": + //TODO (twg) Time Fields ie from/to args + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("no _field present") + } + } + optional = append(optional, fieldName) + sql := fmt.Sprintf(`select distinct field, row from %vλbits where field="%v"`, index, fieldName) + col, ok := call.Args["column"] + if ok { + sql = fmt.Sprintf("%v and column='%v'", sql, col) + } + prev, ok := call.Args["previous"] + if ok { + sql = fmt.Sprintf("%v and row>'%v'", sql, prev) + } + t, ok := call.Args["from"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) + sql += timestampClause + } + t, ok = call.Args["to"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) + sql += timestampClause + } + sql += " order by row" + limit, ok := call.Args["limit"] + if ok { + sql = fmt.Sprintf("%v limit %v", sql, limit) + } + stack = append(stack, sql) + resultType = ARRAY + case "groupby": + //need to figure out how many fields we will be looking at + fields, optColumns, err := FetchFields(call.Children) + panicOn(err) + sql := "select " + for i := range fields { + f := fmt.Sprintf("f%d", i+1) + if i == 0 { + sql += fmt.Sprintf(" %s.field, %s.row", f, f) + } else { + + sql += fmt.Sprintf(",%s.field, %s.row", f, f) + } + } + tname := fmt.Sprintf("%vλbits", index) + sql += fmt.Sprintf(",count(*) as cnt from %v f1", tname) + if len(fields) > 1 { + for i := range fields[1:] { + f := fmt.Sprintf("f%d", i+2) + sql += fmt.Sprintf(" inner join %v %v on f1.column = %v.column", tname, f, f) + } + } + sql += " where " + gb := "" + for i, field := range fields { + f := fmt.Sprintf("f%d", i+1) + if i == 0 { + sql += fmt.Sprintf("%v.field='%v'", f, field) + gb += fmt.Sprintf("%v.field,%v.row", f, f) + } else { + sql += fmt.Sprintf("and %v.field='%v'", f, field) + gb += fmt.Sprintf(",%v.field,%v.row", f, f) + } + } + if len(optColumns) > 0 { + for _, column := range optColumns { + sql += fmt.Sprintf(" and %v ", column) + } + } + sql += " group by " + gb + " order by cnt desc" + + optional = append(optional, len(fields)) + + stack = append(stack, sql) + resultType = GROUPCOUNT + case "topn": + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("no _field present") + } + } + var sq string + if len(call.Children) > 0 { + if len(call.Children) > 1 { + return "", resultType, optional, errors.New("invalid topn") + } + var err error + sq, _, _, err = ToSql(call.Children, index) + panicOn(err) + sq = " and column in (" + sq + ")" + + } + sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) + + limit, ok := call.Args["n"] + if ok { + sql += fmt.Sprintf(" limit %v", limit) + } + stack = append(stack, sql) + resultType = PAIRSFIELD + case "topk": + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("no _field present") + } + } + var sq string + filter, ok := call.Args["filter"] + if ok { + //assuming this works in pilosa so the filter must be a correct bitmap call + //ie row/union/intersect/xor/difference + var err error + sq, _, _, err = ToSql([]*pql.Call{filter.(*pql.Call)}, index) + panicOn(err) + sq = " and column in (" + sq + ")" + + } + t, ok := call.Args["from"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) + sq += timestampClause + } + t, ok = call.Args["to"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) + sq += timestampClause + } + sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) + + limit, ok := call.Args["k"] + if ok { + sql += fmt.Sprintf(" limit %v", limit) + } + stack = append(stack, sql) + resultType = PAIRSFIELD + case "min", "max": + sql, err := getAggregateSql(call, index) + panicOn(err) + stack = append(stack, sql) + resultType = AGGSQL + case "sum": + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + panic("no _field present") + } + } + var sq string + if len(call.Children) > 0 { + //only child should be a bitmap call + var err error + sq, _, _, err = ToSql(call.Children, index) + panicOn(err) + sq = " and column in (" + sq + ")" + + } + sql := fmt.Sprintf(`select sum(val),count(*) from %vλbsi where field="%v"%v`, index, fieldName, sq) + stack = append(stack, sql) + resultType = AGGSQL + case "constrow": + sql := "select column1 as column from (values " + columns, ok := call.Args["columns"].([]interface{}) + + if ok { + for i, col := range columns { + if i != 0 { + sql += "," + } + sql += fmt.Sprintf("(%v)", col) + + } + sql += ")" + + } + + stack = append(stack, sql) + resultType = BITMAP + case "all": + sql := fmt.Sprintf("select column from %vλcolumns", index) + stack = append(stack, sql) + resultType = BITMAP + case "extract": + // extract is a table response + bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) + panicOn(err) + var rowsPart string + if len(call.Children) > 1 { + rows := call.Children[1:] + rowsPart = "select field,row from(" + for i, row := range rows { + part, _, _, err1 := ToSql([]*pql.Call{row}, index) + panicOn(err1) + if i > 0 { + rowsPart += "union all select field,row from (" + } + rowsPart += part + ")" + } + } + var sql string + if len(rowsPart) == 0 { + sql = "select '' as field, '' as row,column from (" + bm + ")" + } else { + sql = "select sq.field, sq.row, b.column from (" + sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) + sql += " where b.column in (" + bm + ")" + } + + stack = append(stack, sql) + resultType = TABLE + case "unionrows": + var rowsPart string + rowsPart = "select field,row from(" + for i, row := range call.Children { + part, _, _, err1 := ToSql([]*pql.Call{row}, index) + panicOn(err1) + if i > 0 { + rowsPart += "union all select field,row from (" + } + rowsPart += part + ")" + } + sql := "select distinct b.column from (" + sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) + stack = append(stack, sql) + resultType = BITMAP + case "includescolumn": + bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) + panicOn(err) + col, ok := call.Args["column"] + if !ok { + return "", resultType, optional, errors.New("missing column parameter") + } + + sql := fmt.Sprintf("select count(*)>0 from (select column from(%v) where column='%v')", bm, col) + stack = append(stack, sql) + resultType = BOOL + default: + goon.Dump(call) + return "", resultType, optional, errors.New(fmt.Sprintf("unknown call '%v'", c)) + } + } + if len(stack) == 0 { + //not sure if i should error on an NOP + return "", resultType, optional, nil + } + return stack[0], resultType, optional, nil +} + +func NewSauron(cfg *SauronConfig) *Sauron { + + addr := "127.0.0.1:8080" + url := "http://" + addr + s := &Sauron{ + cfg: cfg, + addr: addr, + url: url, + sql: NewPQLToSQL(cfg.DatabaseName), + srv: &http.Server{ + Addr: addr, + ReadTimeout: 50 * time.Millisecond, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + }, + } + s.Handler = newRouter(s) + s.srv.Handler = s.Handler + return s +} + +func (p *Sauron) Start() (urlstring string, err error) { + started := make(chan struct{}) + + panicOn(p.sql.Start()) + + // start a pilosa server + bind := p.cfg.Bind + dataDir := p.cfg.DataDir + bindGRPC := p.cfg.BindGRPC + gossipPort := p.cfg.GossipPort + + p.cmd = exec.Command("pilosa", "server", + "--bind", bind, + "--data-dir", dataDir, + "--bind-grpc", bindGRPC, + "--gossip.port", gossipPort) + err = p.cmd.Start() + panicOn(err) + + // wait for it to be listening + p.pilosaHttpURL = "http://" + bind + ok := false + seconds := 20 + for i := 0; i < seconds; i++ { + nc, err := net.Dial("tcp", bind) + if err == nil { + nc.Close() + ok = true + break + } + time.Sleep(time.Second) + } + if !ok { + return "", fmt.Errorf("could not contact pilosa server on %v after %v seconds. pid = %v", bind, seconds, p.cmd.Process.Pid) + } + go func() { + close(started) + + err = p.srv.ListenAndServe() + if err != nil { + if strings.Contains(err.Error(), "Server closed") { + return + } + } + panicOn(err) + }() + <-started + return p.url, nil +} + +func (p *Sauron) Stop() { + panicOn(p.sql.Stop()) + err := p.srv.Shutdown(context.Background()) + panicOn(err) + // Kill pilosa + if err = p.cmd.Process.Kill(); err != nil { + vv("failed to kill process: ", err) + } +} + +// GetAvailPort asks the OS for an unused port. +// There's a race here, where the port could be grabbed by someone else +// before the caller gets to Listen on it, but we are only using +// it to find a random port for the test hang debugging. +// Moreover, in practice such races are rare. Just ask for +// it again if the port is taken. +// Uses net.Listen("tcp", ":0") to determine a free port, then +// releases it back to the OS with Listener.Close(). +func GetAvailPort() int { + l, _ := net.Listen("tcp", ":0") + r := l.Addr() + l.Close() + return r.(*net.TCPAddr).Port +} + +// newRouter creates a new mux http router. +func newRouter(handler *Sauron) http.Handler { + router := mux.NewRouter() + + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") + router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") + router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") + + router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex") + router.HandleFunc("/index/{index}/field/", handler.handlePostField).Methods("POST").Name("PostField") + router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField") + router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring") + router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport") + + return router +} + +// handleGetSchema handles GET /schema requests. +func (s *Sauron) handleGetSchema(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + _ = body + _ = resp + _ = err + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(resp.StatusCode) + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + _, err = w.Write(respBody) + panicOn(err) + + // B) there is no SQL version of get schema. + +} + +// creates an entire schema from the dumped schema. +// requires completed structures, no defaults applied +func (s *Sauron) handlePostSchema(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + + w.WriteHeader(resp.StatusCode) + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + _, err = w.Write(respBody) + panicOn(err) + + // B) SQL + panicOn(s.sql.CreateSchema(body)) +} + +// handlePostField handles /index/{index}/field/{field} requests +func (s *Sauron) handlePostIndex(w http.ResponseWriter, r *http.Request) { + body, resp, err := s.ForwardHandler(w, r) + _ = body + _ = resp + _ = err + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + indexName, ok := mux.Vars(r)["index"] + if !ok { + http.Error(w, "index name is required", http.StatusBadRequest) + return + } + req := struct { + Options pilosa.IndexOptions + }{ + Options: pilosa.IndexOptions{ + Keys: false, + TrackExistence: true, + }} + if len(body) > 0 { //only process if options are present + err = json.Unmarshal(body, &req) + panicOn(err) + } + err = s.sql.CreateIndex(indexName, &req.Options) + panicOn(err) + w.Write(respBody) +} + +// handlePostField handles /index/{index}/field/{field} requests +func (s *Sauron) handlePostField(w http.ResponseWriter, r *http.Request) { + body, resp, err := s.ForwardHandler(w, r) + _ = body + _ = resp + _ = err + indexName, ok := mux.Vars(r)["index"] + if !ok { + http.Error(w, "index name is required", http.StatusBadRequest) + return + } + + fieldName, ok := mux.Vars(r)["field"] + if !ok { + http.Error(w, "field name is required", http.StatusBadRequest) + return + } + + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + // Decode request. + var req postFieldRequest + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + err = dec.Decode(&req) + if err != io.EOF { + panicOn(err) + } + err = s.sql.CreateFieldWithOptions(indexName, fieldName, req.Options) + _, err = w.Write(respBody) + panicOn(err) +} + +// handlePostQuery handles /index/{index}/query requests +func (s *Sauron) handlePostQuery(w http.ResponseWriter, req *http.Request) { + + body, resp, err := s.ForwardHandler(w, req) + pilosaJSON, err := ioutil.ReadAll(resp.Body) + panicOn(err) + var pilosaResponse pilosa.QueryResponse // DEBUG + dec := json.NewDecoder(bytes.NewReader(pilosaJSON)) + err = dec.Decode(&pilosaResponse) + panicOn(err) + qreq := &pilosa.QueryRequest{ + Index: mux.Vars(req)["index"], + Query: string(body), + } + + oracleResponse, err := s.sql.IssueQuery(qreq) + + if !pilosaResponse.Equals(oracleResponse) { + //plan is to write out a useful query response indicating a discrepancy + //TODO (twg) write diff in response somehow + vv("oracle:") + prettyJson(*oracleResponse) + vv("pilosa:") + prettyJson(pilosaResponse) + panic("barf") + } +} +func prettyJson(a pilosa.QueryResponse) { + src, _ := a.MarshalJSON() + dst := &bytes.Buffer{} + if err := json.Indent(dst, src, "", " "); err != nil { + panic(err) + } + vv("%v", string(dst.Bytes())) +} + +func (s *Sauron) FieldInfo(index, field string) (*FieldInfo2, error) { + idx, ok := s.sql.i2f[index] + if ok { + f, ok := idx.Fields[field] + if ok { + return f, nil + } + return nil, errors.New(fmt.Sprintf("field not in schema '%v'", field)) + } + return nil, errors.New(fmt.Sprintf("index not in schema '%v'", index)) +} +func (s *Sauron) handlePostImport(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + _ = body + _ = resp + _ = err + // Get index and field type to determine how to handle the + // import data. + indexName := mux.Vars(req)["index"] + fieldName := mux.Vars(req)["field"] + + _ = indexName + _ = fieldName + + // If the clear flag is true, treat the import as clear bits. + q := req.URL.Query() + doClear := q.Get("clear") == "true" + doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" + _ = doClear + _ = doIgnoreKeyCheck + + info, err := s.FieldInfo(indexName, fieldName) + panicOn(err) + // Unmarshal request based on field type. + fieldType := info.Options.Type + if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal { + iv := &pilosa.ImportValueRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, iv); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + //TODO (twg) ignore columnKeys at the moment + //do importValue + for i, column := range iv.ColumnIDs { + if len(iv.Values) > 0 { + s.sql.Set(indexName, column, fieldName, iv.Values[i], "") + } else if len(iv.FloatValues) > 0 { //TODO (twg) these may not be reachable + s.sql.Set(indexName, column, fieldName, iv.FloatValues[i], "") //probably some issues here + } else if len(iv.StringValues) > 0 { + s.sql.Set(indexName, column, fieldName, iv.StringValues[i], "") + } + } + } else { + // Field type: set, time, mutex + // Marshal into request object. + ir := &pilosa.ImportRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, ir); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + //do import + action := s.sql.Set + if ir.Clear { + action = func(index string, column uint64, field string, value interface{}, ts string) (bool, error) { + return s.sql.Clear(index, column, field, value) + } + } + changed := false + _ = changed + for i, column := range ir.ColumnIDs { + ts := "" + if len(ir.Timestamps) > 0 { + t := time.Unix(ir.Timestamps[i], 0) + ts = t.Format("2006-01-02 15:04:05") + } + if len(ir.RowIDs) > 0 { + change, err := action(indexName, column, fieldName, ir.RowIDs[i], ts) + panicOn(err) + if change { + changed = true + } + } else if len(ir.RowKeys) > 0 { //TODO (twg) these may not be reachable + change, err := action(indexName, column, fieldName, ir.RowKeys[i], ts) + panicOn(err) + if change { + changed = true + } + } + } + } +} +func (s *Sauron) handlePostImportRoaring(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + _ = body + _ = resp + _ = err + // Get index and field type to determine how to handle the + // import data. + indexName := mux.Vars(req)["index"] + fieldName := mux.Vars(req)["field"] + + irr := &pilosa.ImportRoaringRequest{} + err = proto.DefaultSerializer.Unmarshal(body, irr) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + //do i need the shard? + urlVars := mux.Vars(req) + shard, err := strconv.ParseUint(urlVars["shard"], 10, 64) + if err != nil { + panicOn(err) + } + for v, blob := range irr.Views { + _ = v //ignore the view for now + rit, err := roaring.NewRoaringIterator(blob) + panicOn(err) + s.sql.ImportRoaring(indexName, fieldName, shard, rit) + } +} + +//end handler +func (s *Sauron) ForwardHandler(w http.ResponseWriter, req *http.Request) (body []byte, resp *http.Response, err error) { + // TODO (twg) replace all the instances + // we need to buffer the body if we want to read it here and send it + // in the request. + body, err = ioutil.ReadAll(req.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // you can reassign the body if you need to parse it as multipart + req.Body = ioutil.NopCloser(bytes.NewReader(body)) + + // create a new url from the raw RequestURI sent by the client + url := fmt.Sprintf("%s%s", s.pilosaHttpURL, req.RequestURI) + + proxyReq, err := http.NewRequest(req.Method, url, bytes.NewReader(body)) + + // We may want to filter some headers, otherwise we could just use a shallow copy + // proxyReq.Header = req.Header + proxyReq.Header = make(http.Header) + for h, val := range req.Header { + proxyReq.Header[h] = val + } + + resp, err = http.DefaultClient.Do(proxyReq) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + return +} + +// ImportRoaring must not be a bsi field..check higher up the stack +func (ps *PQLToSQL) ImportRoaring(index, field string, shard uint64, itr roaring.RoaringIterator) { + rb := roaring.NewBitmap() + for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { + rb.Put(itrKey, synthC) + } + for _, pos := range rb.Slice() { + row, column := pos/ps.ShardWidth, (shard*ps.ShardWidth)+(pos%ps.ShardWidth) + //ps.SetWithInt64Field(index, column, field, int64(row)) + ps.Set(index, column, field, int64(row), "") + } +} + +func (ps *PQLToSQL) isBSI(index, field string) (full string, isBSI bool) { + idx, ok := ps.i2f[index] + if !ok { + panic(fmt.Errorf("isBSI: no such index '%v'", index)) + } + + fld, ok := idx.Fields[field] + if !ok { + panic(fmt.Errorf("isBSI: no such field '%v'", field)) + } + if fld.Options.Type == "int" { + return dash2theta(index) + "λbsi", true + } + return dash2theta(index) + "λbits", false +} + +// Set is the basic bit operation and value operation as well, depending on the field type +func (ps *PQLToSQL) Set(index string, column uint64, field string, value interface{}, timestamp string) (bool, error) { + table, isBSI := ps.isBSI(index, field) + var sql string + if isBSI { + sql = fmt.Sprintf("INSERT INTO %v VALUES( '%v','%v', '%v')", table, field, column, value) + } else { + //if len(timestamp) == 0 { + // sql = fmt.Sprintf("INSERT INTO %v (field,row,column)VALUES( '%v','%v', '%v')", table, field, value, column) + // } else { + //TODO(twg) will need to make the timestamp as granular as the field is configured. + sql = fmt.Sprintf("INSERT INTO %v (field,row,column,timestamp)VALUES( '%v','%v', '%v','%v')", table, field, value, column, timestamp) + // } + } + + _, err := ps.DB.Exec(sql) + if err != nil { + if sqerr, ok := err.(*sqlite.Error); ok { + if sqerr.Code() == 2067 { //constraint error + return false, nil + } + } + } + panicOn(err) + return true, nil +} + +// Clear removes the specified column or value +func (ps *PQLToSQL) Clear(index string, column uint64, field string, value interface{}) (bool, error) { + table, isBSI := ps.isBSI(index, field) + var sql string + if isBSI { + sql = fmt.Sprintf("delete from %v where field='%v' and column='%v'", table, field, column) + } else { + sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' and column='%v'", table, field, value, column) + } + + res, err := ps.DB.Exec(sql) + panicOn(err) + r, err := res.RowsAffected() + panicOn(err) + if r == 0 { + return false, nil + } + return true, nil +} + +// ClearRow removes entire Row(feature) +func (ps *PQLToSQL) ClearRow(index string, field string, row interface{}) (bool, error) { + table, isBSI := ps.isBSI(index, field) + var sql string + if isBSI { + panic("bsi can't clear row allowed") + } else { + sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' ", table, field, row) + } + + r, err := ps.DB.Exec(sql) + panicOn(err) + n, err := r.RowsAffected() + return n > 0, err +} + +func sliceToString(slc []int64) (r string) { + if len(slc) == 0 { + return + } + for _, i := range slc { + r += fmt.Sprintf("%v,", i) + } + return r[:len(r)-1] +} + +const columnLabel = "col" + +func (ps *PQLToSQL) IssueQuery(req *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { + index := req.Index + q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() + if err != nil { + return nil, errors.Wrap(err, "PQLToSQL.IssueQuery parsing PQL") + } + var results pilosa.QueryResponse + for _, c := range q.Calls { + + if callIndex := c.CallIndex(); callIndex != "" { + index = callIndex + } + + // Handle the field arg. + switch strings.ToLower(c.Name) { + //handle the write queries + //TODO (twg) figure out sqlite changes() function to determine number of bits changed + case "set": + // Read colID. + colID, ok, err := c.UintArg("_" + columnLabel) + if err != nil { + return nil, fmt.Errorf("reading Set() column: %v", err) + } else if !ok { + return nil, fmt.Errorf("Set() column argument '%v' required", columnLabel) + } + ts, ok := c.Args["_timestamp"] + var timestamp string + if ok { + timestamp = strings.Replace(ts.(string), "T", " ", 1) + } + + field, err := c.FieldArg() + if err == nil { + rs, err := ps.Set(index, colID, field, c.Args[field], timestamp) + panicOn(err) + results.Results = append(results.Results, rs) + } else { + return nil, err + } + case "clear": + // Read colID. + colID, ok, err := c.UintArg("_" + columnLabel) + if err != nil { + return nil, fmt.Errorf("reading Clear() column: %v", err) + } else if !ok { + return nil, fmt.Errorf("Clear() column argument '%v' required", columnLabel) + } + + field, err := c.FieldArg() + if err == nil { + rs, err := ps.Clear(index, colID, field, c.Args[field]) + if err != nil { + return nil, fmt.Errorf("Clear() column: %v", err) + } + results.Results = append(results.Results, rs) + } else { + vv("c.FieldArg() returned '%v'", err) + } + case "clearrow": + field, err := c.FieldArg() + panicOn(err) + rs, err := ps.ClearRow(index, field, c.Args[field]) + panicOn(err) + results.Results = append(results.Results, rs) + case "store": + field, err := c.FieldArg() + panicOn(err) + rs, err := ps.Store(index, field, c.Args[field], c.Children) + panicOn(err) + results.Results = append(results.Results, rs) + default: //TODO (twg) move ToSQL into main package + rs, err := ps.readQuery(index, q) + panicOn(err) + results.Results = append(results.Results, rs) + } + } + return &results, nil +} +func (ps *PQLToSQL) handleBitmapCall(sql string) (*pilosa.Row, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + defer rows.Close() + columns := make([]uint64, 0) + for rows.Next() { + //really need a way to know what the results look like + var column uint64 + rows.Scan(&column) + columns = append(columns, column) + } + row := pilosa.NewRow(columns...) + return row, nil +} + +func (ps *PQLToSQL) handleIntCall(sql string) (count uint64, err error) { + rows, err := ps.DB.Query(sql) + if err != nil { + return 0, err + } + defer rows.Close() + for rows.Next() { + //really need a way to know what the results look like + rows.Scan(&count) + return + } + return +} + +type Table struct { +} + +func (ps *PQLToSQL) handleSignedRowCall(sql string) (pilosa.SignedRow, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + + var id uint64 + var pos []uint64 + var neg []uint64 + defer rows.Close() + for rows.Next() { + err := rows.Scan(&id) + if err != nil { + return pilosa.SignedRow{}, err + } + if id >= 0 { + pos = append(pos, id) + } else { + neg = append(neg, id) + } + } + err = rows.Err() + if err != nil { + return pilosa.SignedRow{}, err + } + posRow := pilosa.NewRow(pos...) + negRow := pilosa.NewRow(neg...) + return pilosa.SignedRow{Pos: posRow, Neg: negRow}, nil + +} +func (ps *PQLToSQL) handleStringArrayCall(sql string) ([]interface{}, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + rowKeys := make([]interface{}, 0) + var field string + var id string + defer rows.Close() + for rows.Next() { + err := rows.Scan(&field, &id) + if err != nil { + return nil, err + } + rowKeys = append(rowKeys, id) + } + err = rows.Err() + if err != nil { + return nil, err + } + return rowKeys, nil +} + +func (ps *PQLToSQL) handleIntArrayCall(sql string) (map[string][]interface{}, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + rowIDs := make([]interface{}, 0) + var id uint64 + var field string + defer rows.Close() + for rows.Next() { + err := rows.Scan(&field, &id) + if err != nil { + return nil, err + } + rowIDs = append(rowIDs, id) + } + err = rows.Err() + if err != nil { + return nil, err + } + m := map[string][]interface{}{"rows": rowIDs, "keys": []interface{}{}} + return m, nil +} +func (ps *PQLToSQL) handleGroupCountCall(sqlin string, index string, numFields int) (pilosa.SortedGroupCount, error) { + rows, err := ps.DB.Query(sqlin) + panicOn(err) + //groupCounts := make([]pilosa.GroupCount, 0) + var groupCounts pilosa.SortedGroupCount + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + return nil, err + } + row := make([]interface{}, len(cols)) + for i := range cols { + row[i] = new(sql.RawBytes) + } + for rows.Next() { + // need a row*NumFields + cnt + //err := rows.Scan(&field, &row, &count) + groupCount := pilosa.GroupCount{} + + err := rows.Scan(row...) + if err != nil { + return nil, err + } + for i := 0; i < len(row)-1; i += 2 { + ptr := row[i].(*sql.RawBytes) + fieldName := string(*ptr) + fr := pilosa.FieldRow{Field: fieldName} + fi, err := ps.GetField(index, fieldName) + if err != nil { + return nil, err + } + ptr = row[i+1].(*sql.RawBytes) + v := string(*ptr) + if fi.Options.Keys { + fr.RowKey = v + + } else { + iv, err := strconv.Atoi(v) + if err != nil { + return nil, err + } + fr.RowID = uint64(iv) + } + + groupCount.Group = append(groupCount.Group, fr) + + } + ptr := row[len(row)-1].(*sql.RawBytes) + data := string(*ptr) //convert to string + i, err := strconv.ParseInt(data, 10, 64) + panicOn(err) + groupCount.Count = uint64(i) + groupCounts = append(groupCounts, groupCount) + + } + + err = rows.Err() + if err != nil { + return nil, err + } + return groupCounts, nil +} +func (ps *PQLToSQL) handlePairsFieldCall(sql, index string) (pilosa.PairsField, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + var count uint64 + var row string + defer rows.Close() + res := pilosa.PairsField{} + for rows.Next() { + err := rows.Scan(&row, &count) + if err != nil { + return pilosa.PairsField{}, err + } + pair := pilosa.Pair{Key: row, Count: count} + res.Pairs = append(res.Pairs, pair) + } + err = rows.Err() + if err != nil { + return pilosa.PairsField{}, err + } + return res, nil +} + +// special structures to deal with Extract. In the future I would like to use +// pilosa structs, but dealing with keys makes that complicated for now +// note the column is a string, but i only worry about the type when marshalling out for +// the sqlite part is typeless. Handling keys is almost seemless and not requiring translation +type extractColumns struct { + column string + rows map[int][]interface{} + parent *sauronExtractedIDMatrix +} + +func (eid extractColumns) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + buf.WriteString(fmt.Sprintf(`{"column": %v,"rows":[`, eid.column)) + rows := make([][]interface{}, len(eid.parent.fields)) + for k, v := range eid.rows { + rows[k] = v + } + for i, grow := range rows { + if i != 0 { + buf.WriteByte(44) //write the comma + } + isKey := eid.parent.fields[i].isKey + buf.WriteString(`[`) + for d, row := range grow { + if d != 0 { + buf.WriteByte(44) //write the comma + } + if isKey { + buf.WriteString(fmt.Sprintf(`"%v"`, row)) + } else { + buf.WriteString(fmt.Sprintf(`%v`, row)) + } + + } + buf.WriteString(`]`) + } + + buf.WriteString(`]}`) + return buf.Bytes(), nil +} + +type extractFields struct { + Name string `json:"name"` + Type string `json:"type"` + isKey bool +} +type sauronExtractedIDMatrix struct { + columns []extractColumns + fields []extractFields +} + +func (eid sauronExtractedIDMatrix) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + sort.Slice(eid.columns, func(i, j int) bool { + return eid.columns[i].column < eid.columns[j].column + }) + buf.WriteString(`{"columns":[`) + for i, col := range eid.columns { + b, err := col.MarshalJSON() + panicOn(err) + if i != 0 { + buf.WriteByte(44) //write the comma + } + buf.Write(b) + } + + buf.WriteString(`],"fields":[`) + for i, f := range eid.fields { + if i != 0 { + buf.WriteByte(44) + } + b, err := json.Marshal(f) + panicOn(err) + buf.Write(b) + } + buf.WriteString(`]}`) + return buf.Bytes(), nil +} + +func (ps *PQLToSQL) handleTableCall(sql, index string) (sauronExtractedIDMatrix, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + var field string + var row string + var column string + defer rows.Close() + col := make(map[string]map[string][]interface{}) + fidx := make(map[string]int) + results := sauronExtractedIDMatrix{} + counter := 0 + empty := make(map[string][]interface{}) + for rows.Next() { + err := rows.Scan(&field, &row, &column) + if err != nil { + return sauronExtractedIDMatrix{}, err + } + if field != "" { + _, ok := fidx[field] + if !ok { + fidx[field] = counter + counter++ + } + rows, ok := col[column] + if !ok { + rows = make(map[string][]interface{}) + } + rows[field] = append(rows[field], row) + //results.Columns = append(results.Columns, pilosa.ExtractedIDColumn{ColumnID: column, Rows: [][]uint64{}}) + col[column] = rows + } else { //constrow, or no filed info + col[column] = empty + } + } + err = rows.Err() + if err != nil { + return sauronExtractedIDMatrix{}, err + } + for c, rowMap := range col { + idc := extractColumns{rows: make(map[int][]interface{}), parent: &results} + idc.column = c + for fld, rows := range rowMap { + idx := fidx[fld] + idc.rows[idx] = rows + } + results.columns = append(results.columns, idc) + } + results.fields = make([]extractFields, len(fidx)) + for fieldName, idx := range fidx { + fi, _ := ps.GetField(index, fieldName) + // TODO (twg) hack need to figure out proper way to determine type + aType := "[]uint64" + if fi.Options.Keys { + aType = "[]string" + } + results.fields[idx] = extractFields{Name: fieldName, Type: aType, isKey: fi.Options.Keys} + } + + return results, nil +} + +func (ps *PQLToSQL) handleAggSQLCall(sql, index string) (pilosa.ValCount, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + defer rows.Close() + results := pilosa.ValCount{} + var val int64 + var count int64 + for rows.Next() { + err := rows.Scan(&val, &count) + if err != nil { + return pilosa.ValCount{}, err + } + results.Count = count + results.Val = val + break //shouldn't be anymore rows but being safe + } + err = rows.Err() + if err != nil { + return pilosa.ValCount{}, err + } + return results, nil +} +func (ps *PQLToSQL) handleBoolCall(sql, index string) (bool, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + defer rows.Close() + var val bool + for rows.Next() { + err := rows.Scan(&val) + if err != nil { + return false, err + } + break //shouldn't be anymore rows but being safe + } + err = rows.Err() + if err != nil { + return val, err + } + return val, nil +} + +func (ps *PQLToSQL) readQuery(index string, query *pql.Query) (interface{}, error) { + //Row(color=red) + sql, resultType, optional, err := ToSql(query.Calls, index) + panicOn(err) + switch resultType { + case BITMAP: + return ps.handleBitmapCall(sql) + case INT: + return ps.handleIntCall(sql) + case SIGNEDROW: + return ps.handleSignedRowCall(sql) + case ARRAY: + fieldName := optional[0].(string) + info, ok := ps.i2f[index] + if !ok { + return nil, errors.New(fmt.Sprintf("index not in schema %v", index)) + } + field, ok := info.Fields[fieldName] + if !ok { + return nil, errors.New(fmt.Sprintf("field not in schema %v", fieldName)) + } + if field.Options.Keys { + return ps.handleStringArrayCall(sql) + } else { + return ps.handleIntArrayCall(sql) + } + case GROUPCOUNT: + return ps.handleGroupCountCall(sql, index, optional[0].(int)) + case PAIRSFIELD: + return ps.handlePairsFieldCall(sql, index) + case TABLE: + return ps.handleTableCall(sql, index) + case AGGSQL: + return ps.handleAggSQLCall(sql, index) + case BOOL: + return ps.handleBoolCall(sql, index) + default: + vv("Unknown query Type %v", resultType) + } + return nil, nil +} + +type Int64Slice []int64 + +func (p Int64Slice) Len() int { return len(p) } +func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p Int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +// on tables starting with ps.dbprefix will be returned. +func (ps *PQLToSQL) ListIndexFields() (tables []string) { + for indexName, index := range ps.i2f { + for fieldname, _ := range index.Fields { + + tables = append(tables, indexName+"/"+fieldname) + } + } + sort.Strings(tables) + return tables +} diff --git a/cmd/sauron/populate.sh b/cmd/sauron/populate.sh new file mode 100644 index 000000000..46e7dae67 --- /dev/null +++ b/cmd/sauron/populate.sh @@ -0,0 +1,10 @@ +#!/bin/bash +mod=20 +for i in `seq 100 1`;do +b=$(dc -e "$i $mod %p") +[[ $b == 0 ]] && mod=$(($mod+1)) +r=$(($b+30)) +payload="Set($i, j=$r)" +echo $payload +curl -XPOST "localhost:10101/index/i/query" -d "$payload" +done diff --git a/cmd/sauron/sample_schema.json b/cmd/sauron/sample_schema.json new file mode 100644 index 000000000..4dcc2c501 --- /dev/null +++ b/cmd/sauron/sample_schema.json @@ -0,0 +1 @@ +{"indexes":[{"name":"trait_store","createdAt":1595896639330112903,"options":{"keys":true,"trackExistence":true},"fields":[{"name":"aba","createdAt":1595896641512346986,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"ach_batch_recency","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_pass_thru_recency","createdAt":1595896640333428009,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_payment_recency","createdAt":1595896642230184052,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"bools","createdAt":1595896640151751600,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"bools-exists","createdAt":1595896640855732760,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"central_group","createdAt":1595896640036032355,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"custom_audiences","createdAt":1595896643284443107,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"days_since_last_logon","createdAt":1595896639332677618,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":1,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"db","createdAt":1595896639548206413,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"desktop_recency","createdAt":1595896642633713852,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"domestic_wire_recency","createdAt":1595896642264402807,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"external_transfer_recency","createdAt":1595896639332854139,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"fields","createdAt":1595896639552562813,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"funds_transfer_recency","createdAt":1595896640803192694,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"international_wire_recency","createdAt":1595896643511015744,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"mobile_remote_deposit_recency","createdAt":1595896641174868205,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"payroll_recency","createdAt":1595896642880416952,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"pfm_category_total_current_balance__401k_investment","createdAt":1595896641492935339,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__403b_investment","createdAt":1595896639553865549,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__529_investment","createdAt":1595896643478533291,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__any","createdAt":1595896642202343868,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__auto_loan","createdAt":1595896642260798767,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__brokerage_account","createdAt":1595896639550130078,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__certificate_of_deposit","createdAt":1595896643210723905,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__checking","createdAt":1595896640763548235,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":42,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__credit_card","createdAt":1595896639734800167,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__home_equity_loan","createdAt":1595896642644368353,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__ira_investment","createdAt":1595896642076795921,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__line_of_credit","createdAt":1595896642071432710,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__loan","createdAt":1595896643101755055,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__money_market","createdAt":1595896642574933677,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":36,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__mortgage","createdAt":1595896640934767155,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__personal_loan","createdAt":1595896641378825138,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__roth_ira_investment","createdAt":1595896642821349899,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__savings","createdAt":1595896639534597334,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__simple_ira","createdAt":1595896639952200646,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__student_loan","createdAt":1595896642329394180,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__taxable_investment","createdAt":1595896641406468398,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"phone_recency","createdAt":1595896639708002303,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_count__brokerage_account","createdAt":1595896642501159753,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_cd_or_share_certificate","createdAt":1595896643269910233,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_checking_or_share_draft","createdAt":1595896640044097611,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_credit_card","createdAt":1595896641500609762,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_indirect_auto_loan","createdAt":1595896643353298900,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_line_of_credit","createdAt":1595896641219946523,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_loan","createdAt":1595896641263557904,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_money_market","createdAt":1595896642506172994,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_new_auto_loan","createdAt":1595896640791087766,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_real_estate_loan","createdAt":1595896642616627035,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_savings_or_share","createdAt":1595896642313967229,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_credit_card","createdAt":1595896642254165546,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_loan","createdAt":1595896642870853670,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_checking_or_share_draft","createdAt":1595896639333642847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_credit_card","createdAt":1595896642761030299,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_savings_or_share","createdAt":1595896643227769780,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_vehicle_loan","createdAt":1595896642991171391,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_used_auto_loan","createdAt":1595896642865599530,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_basic_checking_or_share_draft","createdAt":1595896640110059946,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_cd_or_share_certificate","createdAt":1595896640174468431,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_credit_card","createdAt":1595896640192571502,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_indirect_auto_loan","createdAt":1595896640563441892,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896643543919820,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_line_of_credit","createdAt":1595896640188449923,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_loan","createdAt":1595896639761090179,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_money_market","createdAt":1595896642731261748,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_new_auto_loan","createdAt":1595896643143947051,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_checking_or_share_draft","createdAt":1595896640346982139,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_savings_or_share","createdAt":1595896641346726397,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_real_estate_loan","createdAt":1595896643351578694,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_checking_or_share_draft","createdAt":1595896640302899735,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_savings_or_share","createdAt":1595896642726551847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_savings_or_share","createdAt":1595896642961872488,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_credit_card","createdAt":1595896640022927417,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_loan","createdAt":1595896642352438551,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_checking_or_share_draft","createdAt":1595896640804808345,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_credit_card","createdAt":1595896641113696262,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_savings_or_share","createdAt":1595896640312498993,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_vehicle_loan","createdAt":1595896639886308884,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_used_auto_loan","createdAt":1595896642412302198,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__debit_card","createdAt":1595896643127876947,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__education_savings_or_share","createdAt":1595896639338093281,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__escrow_account","createdAt":1595896642969874721,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__health_savings_account","createdAt":1595896640570458731,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_line_of_credit","createdAt":1595896642592520668,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_loan","createdAt":1595896643091627581,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__insurance_product","createdAt":1595896639744476133,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__interest_only_legal_trust_account","createdAt":1595896643005843869,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__membership_share","createdAt":1595896641732585653,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__other_service","createdAt":1595896641807551269,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_401k","createdAt":1595896639333271302,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":11,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_403b","createdAt":1595896643165638942,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_529","createdAt":1595896641640608805,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira","createdAt":1595896639555618615,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira_cd_or_share_certificate","createdAt":1595896641504460031,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__small_business_checking_or_share_draft","createdAt":1595896639332189957,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__special_indirect_vehicle_loan","createdAt":1595896641880199490,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__student_education_loan","createdAt":1595896639551558423,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira","createdAt":1595896643372890708,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira_cd_or_share_certificate","createdAt":1595896641621211665,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__trust","createdAt":1595896640709548779,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__vacation_club_holiday_savings","createdAt":1595896642256997553,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_recency__brokerage_account","createdAt":1595896641955724697,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_cd_or_share_certificate","createdAt":1595896642377298176,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_checking_or_share_draft","createdAt":1595896641640736297,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_credit_card","createdAt":1595896641199968081,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_indirect_auto_loan","createdAt":1595896643314414987,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_line_of_credit","createdAt":1595896641576267257,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_loan","createdAt":1595896641355763147,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_money_market","createdAt":1595896639970740640,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_new_auto_loan","createdAt":1595896642844117593,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_real_estate_loan","createdAt":1595896642953871689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_savings_or_share","createdAt":1595896643325231815,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_credit_card","createdAt":1595896642698454456,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_loan","createdAt":1595896641951407002,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_checking_or_share_draft","createdAt":1595896642711316173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_credit_card","createdAt":1595896642147545663,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_savings_or_share","createdAt":1595896642364919307,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_vehicle_loan","createdAt":1595896639876538030,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_used_auto_loan","createdAt":1595896643527381053,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_basic_checking_or_share_draft","createdAt":1595896639874545938,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_cd_or_share_certificate","createdAt":1595896643147292944,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_credit_card","createdAt":1595896642605033674,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_indirect_auto_loan","createdAt":1595896641097157897,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896642523867033,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_line_of_credit","createdAt":1595896641979726767,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_loan","createdAt":1595896639741458231,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_money_market","createdAt":1595896642350195677,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_new_auto_loan","createdAt":1595896642852653224,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_checking_or_share_draft","createdAt":1595896641391887646,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_savings_or_share","createdAt":1595896643501816261,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_real_estate_loan","createdAt":1595896643015250459,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_checking_or_share_draft","createdAt":1595896640413965217,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_savings_or_share","createdAt":1595896642440765022,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_savings_or_share","createdAt":1595896643523788186,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_credit_card","createdAt":1595896641228514181,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_loan","createdAt":1595896642815200189,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_checking_or_share_draft","createdAt":1595896639648900382,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_credit_card","createdAt":1595896639596071703,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_savings_or_share","createdAt":1595896642447703684,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_vehicle_loan","createdAt":1595896639863224318,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_used_auto_loan","createdAt":1595896639337315659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__debit_card","createdAt":1595896641091084110,"options":{"type":"int","base":0,"bitDepth":1,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__education_savings_or_share","createdAt":1595896643243122929,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__escrow_account","createdAt":1595896642288649376,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__health_savings_account","createdAt":1595896639713687734,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_line_of_credit","createdAt":1595896640644328270,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_loan","createdAt":1595896643005117660,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__insurance_product","createdAt":1595896640080338115,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__interest_only_legal_trust_account","createdAt":1595896640710544689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__membership_share","createdAt":1595896640920327282,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__other_service","createdAt":1595896641868683380,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_401k","createdAt":1595896643214447056,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_403b","createdAt":1595896639331510173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_529","createdAt":1595896641093053659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira","createdAt":1595896640406186395,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira_cd_or_share_certificate","createdAt":1595896642473602012,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__small_business_checking_or_share_draft","createdAt":1595896640631139451,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__special_indirect_vehicle_loan","createdAt":1595896641787887178,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__student_education_loan","createdAt":1595896640496522510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira","createdAt":1595896643383850510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira_cd_or_share_certificate","createdAt":1595896642145329304,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__trust","createdAt":1595896640506789797,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__vacation_club_holiday_savings","createdAt":1595896642206110046,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"stop_payment_recency","createdAt":1595896640954094741,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"survey_2053c210_adfc_4e14_afd3_9abc68a70719","createdAt":1595896642841258189,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2679bd7e_cdb8_4c24_a1da_e904799382eb","createdAt":1595896642498273724,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2f336120_7130_492b_b116_d93d560baa0a","createdAt":1595896639333711539,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_36e229ff_c0a6_48ad_928e_19b5abf69f11","createdAt":1595896642987004213,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa","createdAt":1595896643012024691,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796","createdAt":1595896640184709609,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_587ae933_b8e3_49a2_a23d_989df9ad119f","createdAt":1595896642862965911,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab","createdAt":1595896642292355510,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949","createdAt":1595896640788795677,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_9cebb459_162e_4518_88f6_bc1cdd630cd0","createdAt":1595896642343098690,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_a05de6d3_6620_4eb2_b9a2_511936680b57","createdAt":1595896643389135517,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_b570363c_ba94_4a7a_9891_141a1befc42b","createdAt":1595896643120229702,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_c358d6e0_4116_4230_84ed_869b315e89a1","createdAt":1595896642202109079,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_cce773ce_14e8_4fae_90cd_213a821a3bc0","createdAt":1595896640040535589,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_e49df858_4903_40ac_bd9f_96f764940f2c","createdAt":1595896642346035396,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9","createdAt":1595896640643626147,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"tablet_recency","createdAt":1595896640414042095,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"user_id","createdAt":1595896640846590883,"options":{"type":"int","base":0,"bitDepth":21,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"zip_code","createdAt":1595896640534832822,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}}],"shardWidth":1048576}]} diff --git a/cmd/sauron/sauron_test.go b/cmd/sauron/sauron_test.go new file mode 100644 index 000000000..53cf786de --- /dev/null +++ b/cmd/sauron/sauron_test.go @@ -0,0 +1,396 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strconv" + "testing" + + pb "github.com/golang/protobuf/proto" //nolint:staticcheck + pbuf "github.com/molecula/go-pilosa/v2/gopilosa_pbuf" + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/encoding/proto" + "github.com/pkg/errors" +) + +func TestSauron(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //remove the existing db file if present + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + + schemaJson := `{"indexes":[{"name":"scratch","createdAt":1611185870149882000,"options":{"keys":false,"trackExistence":true}, "fields":[{"name":"luminosity","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}}]}]}` + + client := &http.Client{} + t.Run("Schema", func(t *testing.T) { + _, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) + panicOn(err) + // verify that we can fetch it back + resp, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp.Body) + panicOn(err) + sbody := string(body) + if sbody[:62] != schemaJson[:62] { + fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) + panic("unexpected schema back") + } + }) + // add a set field "color" + //POST localhost:10101/index/scratch/field/color + t.Run("Create Non-Keyed SetField", func(t *testing.T) { + resp, err := client.Post(url+"/index/scratch/field/rating", "application/text", bytes.NewBuffer(nil)) + panicOn(err) + if resp.StatusCode != 200 { + panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) + } + }) + t.Run("Create Keyed SetField", func(t *testing.T) { + resp, err := client.Post(url+"/index/scratch/field/color", "application/text", bytes.NewBuffer([]byte(`{"options": {"keys": true}}`))) + panicOn(err) + if resp.StatusCode != 200 { + panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) + } + }) + t.Run("Create Time Field", func(t *testing.T) { + resp, err := client.Post(url+"/index/scratch/field/event", "application/text", bytes.NewBuffer([]byte(`{"options": { "type": "time", "timeQuantum": "YMDH" }}`))) + panicOn(err) + if resp.StatusCode != 200 { + panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) + } + }) + + // set some bits + + // set some bits + + t.Run("Set", func(t *testing.T) { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) + panicOn(err) + // test for dups + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) + panicOn(err) + // + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=red)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=green)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=green)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=green)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=green)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=yellow)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=yellow)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=yellow)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=yellow)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=orange)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=orange)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=orange)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=orange)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, luminosity=1)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, luminosity=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, luminosity=4)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, luminosity=5)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, luminosity=4)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, luminosity=3)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, luminosity=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, luminosity=1)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, rating=1)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(10, rating=1)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, rating=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, rating=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, rating=3)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, rating=4)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, rating=4)`)) + panicOn(err) + + //time fields + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T01:00)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T02:00)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, event=1,2021-02-05T02:05)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, event=1,2021-02-05T03:00)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, event=1,2021-02-05T04:00)`)) + panicOn(err) + }) + t.Run("Clear", func(t *testing.T) { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Clear(1, color=purple)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`ClearRow(color=purple)`)) + panicOn(err) + }) + t.Run("Store", func(t *testing.T) { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Store(Row(color=red), color=purple)`)) + panicOn(err) + }) + // verify that we can fetch it back + var schema pilosa.Schema + t.Run("Fetch Schema", func(t *testing.T) { + resp, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp.Body) + panicOn(err) + err = json.Unmarshal(body, &schema) + if err != nil { + panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) + } + }) + t.Run("Import Roaring", func(t *testing.T) { + //sets 12 bits in shard 0 + roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + idx := schema.Indexes[0] + var fld *pilosa.FieldInfo + for _, f := range idx.Fields { + if f.Name == "rating" { + fld = f + } + } + if fld == nil { + panic("field color not found") + } + msg := pilosa.ImportRoaringRequest{ + IndexCreatedAt: idx.CreatedAt, + FieldCreatedAt: fld.CreatedAt, + Clear: false, + Views: map[string][]byte{ + "": roaringData, + }, + UpdateExistence: true, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + furl := url + "/index/scratch/field/rating/import-roaring/0" + req, err := http.NewRequest("POST", furl, bytes.NewBuffer(data)) + panicOn(err) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + + // maybe need to Accept: "application/x-protobuf" + resp, err := client.Do(req) + panicOn(err) + _ = resp + }) + t.Run("Read Query", func(t *testing.T) { + for _, tst := range GetTests() { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(tst.Pql)) + panicOn(err) + } + }) +} + +func makeImportRequest(field *FieldInfo2, shard uint64, rows, columns []uint64, clear bool) (path string, data []byte, err error) { + msg := &pbuf.ImportRequest{ + Index: field.GetIndexName(), + IndexCreatedAt: field.GetIndexCreatedAt(), + Field: field.GetName(), + FieldCreatedAt: field.GetCreatedAt(), + Shard: shard, + RowIDs: rows, + ColumnIDs: columns, + } + data, err = pb.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling Import to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) + return path, data, nil +} +func makeImportKeyRequest(field *FieldInfo2, shard uint64, rows []string, columns []uint64, clear bool) (path string, data []byte, err error) { + msg := &pbuf.ImportRequest{ + Index: field.GetIndexName(), + IndexCreatedAt: field.GetIndexCreatedAt(), + Field: field.GetName(), + FieldCreatedAt: field.GetCreatedAt(), + Shard: shard, + RowKeys: rows, + ColumnIDs: columns, + } + data, err = pb.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling Import to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) + return path, data, nil +} + +func makeImportValueRequest(field *FieldInfo2, shard uint64, values []int64, columns []uint64, clear bool) (path string, data []byte, err error) { + msg := &pbuf.ImportValueRequest{ + Index: field.GetIndexName(), + IndexCreatedAt: field.GetIndexCreatedAt(), + Field: field.GetName(), + FieldCreatedAt: field.GetCreatedAt(), + Shard: shard, + Values: values, + ColumnIDs: columns, + } + data, err = pb.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling Import to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) + return path, data, nil +} +func TestSauronImport(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //remove the existing db file if present + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + schemaJson := `{ + "indexes": [{ + "name": "scratch", + "createdAt": 1611185870149882000, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [{ + "name": "bsi", + "createdAt": 1595896639332730413, + "options": { + "type": "int", + "base": 0, + "bitDepth": 31, + "min": -9223372036854775808, + "max": 9223372036854775807, + "keys": false, + "foreignIndex": "" + } + }, { + "name": "decimal", + "createdAt": 1613177295654228860, + "options": { + "type": "decimal", + "base": 0, + "scale": 1, + "bitDepth": 0, + "min": -922337203685477580.8, + "max": 922337203685477580.7, + "keys": false + } + },{ + "name": "setkeyfield", + "createdAt": 1613345793598357200, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + },{ + "name": "setfield", + "createdAt": 1613345793598357200, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }] + }] +}` + client := &http.Client{} + _, err = client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) + panicOn(err) + + t.Run("basic row/column", func(t *testing.T) { + f, err := proxy.sql.GetField("scratch", "setfield") + panicOn(err) + clear := false + cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} + rows := []uint64{0, 0, 0, 1, 1, 1, 2, 2, 2} + shard := uint64(0) + path, payload, err := makeImportRequest(f, shard, rows, cols, clear) + _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) + panicOn(err) + }) + t.Run("basic rowkey/column", func(t *testing.T) { + f, err := proxy.sql.GetField("scratch", "setkeyfield") + panicOn(err) + clear := false + cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} + rows := []string{"red", "red", "red", "blue", "blue", "blue", "green", "green", "green"} + shard := uint64(0) + path, payload, err := makeImportKeyRequest(f, shard, rows, cols, clear) + _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) + panicOn(err) + }) + t.Run("basic import values", func(t *testing.T) { + f, err := proxy.sql.GetField("scratch", "bsi") + panicOn(err) + clear := false + cols := []uint64{0, 1, 2, 3, 4} + values := []int64{40, 30, 20, 10, 5} + shard := uint64(0) + path, payload, err := makeImportValueRequest(f, shard, values, cols, clear) + _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) + panicOn(err) + }) + +} diff --git a/cmd/sauron/schema_test.go b/cmd/sauron/schema_test.go new file mode 100644 index 000000000..23f5e924b --- /dev/null +++ b/cmd/sauron/schema_test.go @@ -0,0 +1,353 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/pilosa/pilosa/v2" +) + +func NewSauronTestConfig(t *testing.T) *SauronConfig { + + bind := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) + bindGRPC := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) + gossipPort := fmt.Sprintf("%d", GetAvailPort()) + dataDir, err := ioutil.TempDir("", "pilosa-sauron-target-*") + panicOn(err) + + return &SauronConfig{ + DatabaseName: strings.ToLower(t.Name()), + Bind: bind, + BindGRPC: bindGRPC, + GossipPort: gossipPort, + DataDir: dataDir, + } +} + +func TestProxyCanHandlePostSchema(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //start with + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + + // prep for test by cleaning up leftovers from any old run. + //panicOn(proxy.sql.DropTablesWithPrefix(t.Name())) + + //schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } }, { "name ": "color", "options ": {} } ] }] }` + /* + schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } } ] }] }` + + + vv("Posting err = '%v'", schemaJson) + resp, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) + if resp.StatusCode != http.StatusNoContent { + panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) + } + // verify that we can fetch it back + resp2, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp2.Body) + panicOn(err) + sbody := string(body) + vv("body = '%v'", sbody) + if sbody[:62] != schemaJson[:62] { + fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) + panic("unxpected schema back") + } + */ + // load large schema too + client := &http.Client{} + schm, err := ioutil.ReadFile("sample_schema.json") + panicOn(err) + + resp, err := client.Post(url+"/schema", "application/json", bytes.NewBuffer(schm)) + panicOn(err) + if resp.StatusCode != http.StatusNoContent { + panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) + } + + // verify all tables expected are present + obs := make(map[string]bool) + for i, table := range proxy.sql.ListIndexFields() { + _ = i + obs[table] = true + if !expectedTables[table] { + panic(fmt.Sprintf("observed table '%v' but was not expected", table)) + } + } + for table := range expectedTables { + if !obs[table] { + panic(fmt.Sprintf("expected table '%v' but was not observed", table)) + } + } + +} + +func TestProxyCanCreateIndexFieldViaPost(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //start with + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + client := &http.Client{} + //create non-keyed or "simple" index + resp, err := client.Post(url+"/index/simple", "application/json", bytes.NewBuffer(nil)) + panicOn(err) + if resp.StatusCode != http.StatusOK { + panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) + } + + fields := []struct { + name string + options []byte + }{ + {name: "set", options: []byte{}}, + {name: "keyset", options: []byte{}}, + {name: "bsi", options: []byte{}}, + } + for _, field := range fields { + //create set field + resp, err = client.Post(fmt.Sprintf("%v/index/simple/field/%v", url, field.name), "application/json", bytes.NewBuffer(field.options)) + panicOn(err) + if resp.StatusCode != http.StatusOK { + panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) + } + } + // verify that we can fetch it back + resp2, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp2.Body) + panicOn(err) + var schema pilosa.Schema + err = json.Unmarshal(body, &schema) + if err != nil { + panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) + } + + expected := map[string]map[string]bool{ + "simple": {"bsi": true, "set": true, "keyset": true}, + } + for _, i := range schema.Indexes { + ll, ok := expected[i.Name] + if !ok { + panic(fmt.Sprintf("expected index not present: %v", i.Name)) + } + for _, f := range i.Fields { + _, ok := ll[f.Name] + if !ok { + panic(fmt.Sprintf("expected field '%v' not present in index'%v'", f.Name, i.Name)) + } + } + } +} + +var expectedTables = map[string]bool{ + "trait_store/aba": true, + "trait_store/ach_batch_recency": true, + "trait_store/ach_pass_thru_recency": true, + "trait_store/ach_payment_recency": true, + "trait_store/bools": true, + "trait_store/bools-exists": true, + "trait_store/central_group": true, + "trait_store/custom_audiences": true, + "trait_store/days_since_last_logon": true, + "trait_store/db": true, + "trait_store/desktop_recency": true, + "trait_store/domestic_wire_recency": true, + "trait_store/external_transfer_recency": true, + "trait_store/fields": true, + "trait_store/funds_transfer_recency": true, + "trait_store/international_wire_recency": true, + "trait_store/mobile_remote_deposit_recency": true, + "trait_store/payroll_recency": true, + "trait_store/pfm_category_total_current_balance__401k_investment": true, + "trait_store/pfm_category_total_current_balance__403b_investment": true, + "trait_store/pfm_category_total_current_balance__529_investment": true, + "trait_store/pfm_category_total_current_balance__any": true, + "trait_store/pfm_category_total_current_balance__auto_loan": true, + "trait_store/pfm_category_total_current_balance__brokerage_account": true, + "trait_store/pfm_category_total_current_balance__certificate_of_deposit": true, + "trait_store/pfm_category_total_current_balance__checking": true, + "trait_store/pfm_category_total_current_balance__credit_card": true, + "trait_store/pfm_category_total_current_balance__home_equity_loan": true, + "trait_store/pfm_category_total_current_balance__ira_investment": true, + "trait_store/pfm_category_total_current_balance__line_of_credit": true, + "trait_store/pfm_category_total_current_balance__loan": true, + "trait_store/pfm_category_total_current_balance__money_market": true, + "trait_store/pfm_category_total_current_balance__mortgage": true, + "trait_store/pfm_category_total_current_balance__personal_loan": true, + "trait_store/pfm_category_total_current_balance__roth_ira_investment": true, + "trait_store/pfm_category_total_current_balance__savings": true, + "trait_store/pfm_category_total_current_balance__simple_ira": true, + "trait_store/pfm_category_total_current_balance__student_loan": true, + "trait_store/pfm_category_total_current_balance__taxable_investment": true, + "trait_store/phone_recency": true, + "trait_store/product_count__brokerage_account": true, + "trait_store/product_count__commercial_cd_or_share_certificate": true, + "trait_store/product_count__commercial_checking_or_share_draft": true, + "trait_store/product_count__commercial_credit_card": true, + "trait_store/product_count__commercial_indirect_auto_loan": true, + "trait_store/product_count__commercial_line_of_credit": true, + "trait_store/product_count__commercial_loan": true, + "trait_store/product_count__commercial_money_market": true, + "trait_store/product_count__commercial_new_auto_loan": true, + "trait_store/product_count__commercial_real_estate_loan": true, + "trait_store/product_count__commercial_savings_or_share": true, + "trait_store/product_count__commercial_secured_credit_card": true, + "trait_store/product_count__commercial_secured_loan": true, + "trait_store/product_count__commercial_special_checking_or_share_draft": true, + "trait_store/product_count__commercial_special_credit_card": true, + "trait_store/product_count__commercial_special_savings_or_share": true, + "trait_store/product_count__commercial_special_vehicle_loan": true, + "trait_store/product_count__commercial_used_auto_loan": true, + "trait_store/product_count__consumer_basic_checking_or_share_draft": true, + "trait_store/product_count__consumer_cd_or_share_certificate": true, + "trait_store/product_count__consumer_credit_card": true, + "trait_store/product_count__consumer_indirect_auto_loan": true, + "trait_store/product_count__consumer_interest_bearing_checking_or_share_draft": true, + "trait_store/product_count__consumer_line_of_credit": true, + "trait_store/product_count__consumer_loan": true, + "trait_store/product_count__consumer_money_market": true, + "trait_store/product_count__consumer_new_auto_loan": true, + "trait_store/product_count__consumer_premium_checking_or_share_draft": true, + "trait_store/product_count__consumer_premium_savings_or_share": true, + "trait_store/product_count__consumer_real_estate_loan": true, + "trait_store/product_count__consumer_restricted_checking_or_share_draft": true, + "trait_store/product_count__consumer_restricted_savings_or_share": true, + "trait_store/product_count__consumer_savings_or_share": true, + "trait_store/product_count__consumer_secured_credit_card": true, + "trait_store/product_count__consumer_secured_loan": true, + "trait_store/product_count__consumer_special_checking_or_share_draft": true, + "trait_store/product_count__consumer_special_credit_card": true, + "trait_store/product_count__consumer_special_savings_or_share": true, + "trait_store/product_count__consumer_special_vehicle_loan": true, + "trait_store/product_count__consumer_used_auto_loan": true, + "trait_store/product_count__debit_card": true, + "trait_store/product_count__education_savings_or_share": true, + "trait_store/product_count__escrow_account": true, + "trait_store/product_count__health_savings_account": true, + "trait_store/product_count__home_equity_line_of_credit": true, + "trait_store/product_count__home_equity_loan": true, + "trait_store/product_count__insurance_product": true, + "trait_store/product_count__interest_only_legal_trust_account": true, + "trait_store/product_count__membership_share": true, + "trait_store/product_count__other_service": true, + "trait_store/product_count__plan_401k": true, + "trait_store/product_count__plan_403b": true, + "trait_store/product_count__plan_529": true, + "trait_store/product_count__roth_ira": true, + "trait_store/product_count__roth_ira_cd_or_share_certificate": true, + "trait_store/product_count__small_business_checking_or_share_draft": true, + "trait_store/product_count__special_indirect_vehicle_loan": true, + "trait_store/product_count__student_education_loan": true, + "trait_store/product_count__traditional_ira": true, + "trait_store/product_count__traditional_ira_cd_or_share_certificate": true, + "trait_store/product_count__trust": true, + "trait_store/product_count__vacation_club_holiday_savings": true, + "trait_store/product_recency__brokerage_account": true, + "trait_store/product_recency__commercial_cd_or_share_certificate": true, + "trait_store/product_recency__commercial_checking_or_share_draft": true, + "trait_store/product_recency__commercial_credit_card": true, + "trait_store/product_recency__commercial_indirect_auto_loan": true, + "trait_store/product_recency__commercial_line_of_credit": true, + "trait_store/product_recency__commercial_loan": true, + "trait_store/product_recency__commercial_money_market": true, + "trait_store/product_recency__commercial_new_auto_loan": true, + "trait_store/product_recency__commercial_real_estate_loan": true, + "trait_store/product_recency__commercial_savings_or_share": true, + "trait_store/product_recency__commercial_secured_credit_card": true, + "trait_store/product_recency__commercial_secured_loan": true, + "trait_store/product_recency__commercial_special_checking_or_share_draft": true, + "trait_store/product_recency__commercial_special_credit_card": true, + "trait_store/product_recency__commercial_special_savings_or_share": true, + "trait_store/product_recency__commercial_special_vehicle_loan": true, + "trait_store/product_recency__commercial_used_auto_loan": true, + "trait_store/product_recency__consumer_basic_checking_or_share_draft": true, + "trait_store/product_recency__consumer_cd_or_share_certificate": true, + "trait_store/product_recency__consumer_credit_card": true, + "trait_store/product_recency__consumer_indirect_auto_loan": true, + "trait_store/product_recency__consumer_interest_bearing_checking_or_share_draft": true, + "trait_store/product_recency__consumer_line_of_credit": true, + "trait_store/product_recency__consumer_loan": true, + "trait_store/product_recency__consumer_money_market": true, + "trait_store/product_recency__consumer_new_auto_loan": true, + "trait_store/product_recency__consumer_premium_checking_or_share_draft": true, + "trait_store/product_recency__consumer_premium_savings_or_share": true, + "trait_store/product_recency__consumer_real_estate_loan": true, + "trait_store/product_recency__consumer_restricted_checking_or_share_draft": true, + "trait_store/product_recency__consumer_restricted_savings_or_share": true, + "trait_store/product_recency__consumer_savings_or_share": true, + "trait_store/product_recency__consumer_secured_credit_card": true, + "trait_store/product_recency__consumer_secured_loan": true, + "trait_store/product_recency__consumer_special_checking_or_share_draft": true, + "trait_store/product_recency__consumer_special_credit_card": true, + "trait_store/product_recency__consumer_special_savings_or_share": true, + "trait_store/product_recency__consumer_special_vehicle_loan": true, + "trait_store/product_recency__consumer_used_auto_loan": true, + "trait_store/product_recency__debit_card": true, + "trait_store/product_recency__education_savings_or_share": true, + "trait_store/product_recency__escrow_account": true, + "trait_store/product_recency__health_savings_account": true, + "trait_store/product_recency__home_equity_line_of_credit": true, + "trait_store/product_recency__home_equity_loan": true, + "trait_store/product_recency__insurance_product": true, + "trait_store/product_recency__interest_only_legal_trust_account": true, + "trait_store/product_recency__membership_share": true, + "trait_store/product_recency__other_service": true, + "trait_store/product_recency__plan_401k": true, + "trait_store/product_recency__plan_403b": true, + "trait_store/product_recency__plan_529": true, + "trait_store/product_recency__roth_ira": true, + "trait_store/product_recency__roth_ira_cd_or_share_certificate": true, + "trait_store/product_recency__small_business_checking_or_share_draft": true, + "trait_store/product_recency__special_indirect_vehicle_loan": true, + "trait_store/product_recency__student_education_loan": true, + "trait_store/product_recency__traditional_ira": true, + "trait_store/product_recency__traditional_ira_cd_or_share_certificate": true, + "trait_store/product_recency__trust": true, + "trait_store/product_recency__vacation_club_holiday_savings": true, + "trait_store/stop_payment_recency": true, + "trait_store/survey_2053c210_adfc_4e14_afd3_9abc68a70719": true, + "trait_store/survey_2679bd7e_cdb8_4c24_a1da_e904799382eb": true, + "trait_store/survey_2f336120_7130_492b_b116_d93d560baa0a": true, + "trait_store/survey_36e229ff_c0a6_48ad_928e_19b5abf69f11": true, + "trait_store/survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa": true, + "trait_store/survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796": true, + "trait_store/survey_587ae933_b8e3_49a2_a23d_989df9ad119f": true, + "trait_store/survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab": true, + "trait_store/survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949": true, + "trait_store/survey_9cebb459_162e_4518_88f6_bc1cdd630cd0": true, + "trait_store/survey_a05de6d3_6620_4eb2_b9a2_511936680b57": true, + "trait_store/survey_b570363c_ba94_4a7a_9891_141a1befc42b": true, + "trait_store/survey_c358d6e0_4116_4230_84ed_869b315e89a1": true, + "trait_store/survey_cce773ce_14e8_4fae_90cd_213a821a3bc0": true, + "trait_store/survey_e49df858_4903_40ac_bd9f_96f764940f2c": true, + "trait_store/survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9": true, + "trait_store/tablet_recency": true, + "trait_store/user_id": true, + "trait_store/zip_code": true, +} diff --git a/cmd/sauron/sqlgen_test.go b/cmd/sauron/sqlgen_test.go new file mode 100644 index 000000000..b9d11a728 --- /dev/null +++ b/cmd/sauron/sqlgen_test.go @@ -0,0 +1,260 @@ +package main + +import ( + "strings" + "testing" + + "github.com/pilosa/pilosa/v2/pql" + "github.com/shurcooL/go-goon" +) + +/* +sqlite db this test validated this against +create table bits (field ,row , column , timestamp); + create table λbsi (field string,column bigint , val bigint); + CREATE VIEW columns as + select distinct column from λbits + UNION + select distinct column from λbsi; + + INSERT INTO λbits VALUES( 'color','red',1); + INSERT INTO λbits VALUES( 'color','red',2); + INSERT INTO λbits VALUES( 'color','red',3); + INSERT INTO λbits VALUES( 'color','red',4); + INSERT INTO λbits VALUES( 'color','red',5); + INSERT INTO λbits VALUES( 'color','green',2); + INSERT INTO λbits VALUES( 'color','green',4); + INSERT INTO λbits VALUES( 'color','green',6); + INSERT INTO λbits VALUES( 'color','green',8); + INSERT INTO λbits VALUES( 'color','yellow',1); + INSERT INTO λbits VALUES( 'color','yellow',3); + INSERT INTO λbits VALUES( 'color','yellow',5); + INSERT INTO λbits VALUES( 'color','yellow',7); + INSERT INTO λbits VALUES( 'color','orange',5); + INSERT INTO λbits VALUES( 'color','orange',6); + INSERT INTO λbits VALUES( 'color','orange',7); + INSERT INTO λbits VALUES( 'color','orange',8); + + INSERT INTO λbits VALUES( 'rating',1,8); + INSERT INTO λbits VALUES( 'rating',1,7); + INSERT INTO λbits VALUES( 'rating',1,10); + INSERT INTO λbits VALUES( 'rating',2,6); + INSERT INTO λbits VALUES( 'rating',2,5); + INSERT INTO λbits VALUES( 'rating',3,4); + INSERT INTO λbits VALUES( 'rating',4,3); + INSERT INTO λbits VALUES( 'rating',4,2); + + INSERT INTO λbsi VALUES( 'luminosity',1, 1); + INSERT INTO λbsi VALUES( 'luminosity',2, 2); + INSERT INTO λbsi VALUES( 'luminosity',3, 4); + INSERT INTO λbsi VALUES( 'luminosity',4, 5); + INSERT INTO λbsi VALUES( 'luminosity',5, 4); + INSERT INTO λbsi VALUES( 'luminosity',6, 3); + INSERT INTO λbsi VALUES( 'luminosity',7, 2); + INSERT INTO λbsi VALUES( 'luminosity',8, 1); +*/ +func GetTests() []struct { + Pql string + Sql string +} { + return []struct { + Pql string + Sql string + }{ + { + Pql: "Row(rating=0)", + Sql: `select distinct column from λbits where field="rating" AND row="0"`, + }, + { + Pql: "Row(color=red)", + Sql: `select distinct column from λbits where field="color" AND row="red"`, + }, + { + Pql: "Row(luminosity>2)", + Sql: `select distinct column from λbsi where field="luminosity" and val>2`, + }, + { + Pql: "Row(luminosity<2)", + Sql: `select distinct column from λbsi where field="luminosity" and val<2`, + }, + { + Pql: "Row(luminosity==4)", + Sql: `select distinct column from λbsi where field="luminosity" and val=4`, + }, + { + Pql: "Row(luminosity<=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val<=4`, + }, + { + Pql: "Row(luminosity>=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val>=4`, + }, + { + Pql: "Row(luminosity!=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val!=4`, + }, + { + Pql: "Row(2<=luminosity<=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val>=2 and val<=4`, + }, + { + Pql: "Row(22 and val<=4`, + }, + { + Pql: "Row(22 and val<4`, + }, + { + Pql: `Row(event=1, from='2021-02-01T00:00', to='2021-02-05T03:00')`, + Sql: `select distinct column from λbits where field="event" AND row="1" AND timestamp >= "2021-02-01 00:00" AND timestamp < "2021-02-05 03:00"`, + }, + { + Pql: "Count(row(color=red))", + Sql: `select count(*) from( select distinct column from λbits where field="color" AND row="red" )`, + }, + { + Pql: `Intersect(Row(color=red),Row(color=green))`, + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: "Count(Union(Row(color=red),Row(color=green)))", + Sql: `select count(*) from( select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") )`, + }, + { + Pql: "Union(Row(color=red),Row(color=green))", + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: "Difference(Row(color=red),Row(color=green))", + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" except select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: `All()`, + Sql: `select column from λcolumns`, + }, + { + Pql: "Not(Row(color=red))", + Sql: `select column from λcolumns except select distinct column from λbits where field="color" AND row="red"`, + }, + { + Pql: "Xor(Row(color=red),Row(color=green))", + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") except select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: "Distinct(field=luminosity)", + Sql: `select distinct val from λbsi where field="luminosity"`, + }, + { + Pql: "Distinct(Row(color=red),field=luminosity)", + Sql: `select distinct val from λbsi where field="luminosity" AND column in (select distinct column from λbits where field="color" AND row="red")`, + }, + { + Pql: "Distinct(Intersect(Row(color=red),Row(color=green)),field=luminosity)", + Sql: `select distinct val from λbsi where field="luminosity" AND column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green"))`, + }, + { + Pql: "Rows(color)", + Sql: `select distinct field, row from λbits where field="color" order by row`, + }, + { + Pql: "Rows(color, limit=10)", + Sql: `select distinct field, row from λbits where field="color" order by row limit 10`, + }, + { + Pql: "Rows(color, column=1)", + Sql: `select distinct field, row from λbits where field="color" and column='1' order by row`, + }, + // { // TODO (twg) Not supported by this test, can't reliably get key ordering + // Pql: "Rows(color, previous=green )", + // Sql: `select distinct field, row from λbits where field="color" and row>'green' order by row`, + // }, + { + Pql: "GroupBy(Rows(color))", + Sql: `select f1.field, f1.row,count(*) as cnt from λbits f1 where f1.field='color' group by f1.field,f1.row order by cnt desc`, + }, + { + Pql: "GroupBy(Rows(color),Rows(rating))", + Sql: `select f1.field, f1.row,f2.field, f2.row,count(*) as cnt from λbits f1 inner join λbits f2 on f1.column = f2.column where f1.field='color'and f2.field='rating' group by f1.field,f1.row,f2.field,f2.row order by cnt desc`, + }, + { + Pql: "TopN(color)", + Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, + }, + { + Pql: "TopN(color,n=2)", + Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc limit 2`, + }, + { + Pql: "TopN(color,Row(color=red),n=2)", + Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="color" AND row="red") group by row order by cnt desc limit 2`, + }, + { + Pql: "Topk(color)", + Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, + }, + { + Pql: "TopK(color, k=2, filter=Row(rating=1))", + Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="rating" AND row="1") group by row order by cnt desc limit 2`, + }, + { + Pql: `Min(field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity")`, + }, + { + Pql: `Max(field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select max(val) from λbsi where field="luminosity")`, + }, + { + Pql: `Min(Row(color=orange),field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select distinct column from λbits where field="color" AND row="orange")) and column in (select distinct column from λbits where field="color" AND row="orange")`, + }, + { + Pql: `Min(Intersect(Row(color=red),Row(color=orange)),field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))) and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))`, + }, + { + Pql: `Sum(field="luminosity")`, + Sql: `select sum(val),count(*) from λbsi where field="luminosity"`, + }, + { + Pql: `ConstRow(columns=[1,3,5])`, + Sql: `select column1 as column from (values (1),(3),(5))`, + }, + { + Pql: `UnionRows(Rows(color))`, + Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, + }, + { + Pql: `UnionRows(Rows(color),Rows(rating))`, + Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, + }, + { + Pql: `IncludesColumn(Row(color=red), column=1)`, + Sql: `select count(*)>0 from (select column from(select distinct column from λbits where field="color" AND row="red") where column='1')`, + }, + { + Pql: `Extract(Row(color=red),Rows(color),Rows(rating))`, + Sql: `select sq.field, sq.row, b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row where b.column in (select distinct column from λbits where field="color" AND row="red")`, + }, + { + Pql: `Extract(ConstRow(columns=[1,2,3]))`, + Sql: `select '' as field, '' as row,column from (select column1 as column from (values (1),(2),(3)))`, + }, + } +} +func TestParse(t *testing.T) { + + for _, tst := range GetTests() { + + q, err := pql.NewParser(strings.NewReader(tst.Pql)).Parse() + panicOn(err) + sql, _, _, err := ToSql(q.Calls, "") + panicOn(err) + if sql != tst.Sql { + vv("\npql:%v\ngenerated:\n%v\nexpected:\n%v\n", tst.Pql, sql, tst.Sql) + goon.Dump(q) + } + + } +} diff --git a/cmd/sauron/test_sqlite.sh b/cmd/sauron/test_sqlite.sh new file mode 100755 index 000000000..d0ab85b2f --- /dev/null +++ b/cmd/sauron/test_sqlite.sh @@ -0,0 +1,51 @@ +#!/bin/bash +db="poc2.db" +sqlite3 ${db} < 0 { + frames := runtime.CallersFrames(pc[:n]) + for i := 0; i <= target; i++ { + contender, more := frames.Next() + if i == target { + f = contender + } + if !more { + break + } + } + } + return f.Function +} From 6219b4ca8b9cf84c66decdacc61dc1becca29b28 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Feb 2021 17:47:28 -0600 Subject: [PATCH 02/28] add optional UpdateExistence on importRoaring --- api.go | 42 +++++-- encoding/proto/proto.go | 14 ++- handler.go | 13 +- http/client_test.go | 44 +++++++ internal/public.pb.go | 265 +++++++++++++++++++++++----------------- internal/public.proto | 1 + pilosa_internal_test.go | 45 ++++++- roaring/roaring.go | 41 +++++++ 8 files changed, 325 insertions(+), 140 deletions(-) diff --git a/api.go b/api.go index 55ea78af4..224c6bf15 100644 --- a/api.go +++ b/api.go @@ -17,6 +17,7 @@ package pilosa import ( + "bytes" "context" "encoding/binary" "encoding/csv" @@ -421,22 +422,25 @@ func importWorker(importWork chan importJob) { fallthrough case RequestActionSet: fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) - if fileMagic == roaring.MagicNumber { // if pilosa roaring format - err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear) - if err != nil { - return errors.Wrap(err, "importing pilosa roaring") - } - } else { + data := viewData + if fileMagic != roaring.MagicNumber { // if pilosa roaring format // must make a copy of data to operate on locally on standard roaring format. // field.importRoaring changes the standard roaring run format to pilosa roaring - data := make([]byte, len(viewData)) + data = make([]byte, len(viewData)) copy(data, viewData) - err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) - - if err != nil { - return errors.Wrap(err, "importing standard roaring") + } + if j.req.UpdateExistence { + if ef := j.field.idx.existenceField(); ef != nil { + existence := combineForExistence(data) + ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false) } } + + err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) + + if err != nil { + return errors.Wrap(err, "importing standard roaring") + } } return nil }(); err != nil { @@ -453,6 +457,22 @@ func importWorker(importWork chan importJob) { } } +// merge all rows to singled existence row +func combineForExistence(inputRoaringData []byte) []byte { + rowSize := uint64(1 << shardVsContainerExponent) + rit, err := roaring.NewRoaringIterator(inputRoaringData) + if err != nil { + panicOn(err) + } + bm := roaring.NewBitmap() + bm.MergeRoaringRawIteratorIntoExists(rit, rowSize) + buf := new(bytes.Buffer) + + _, err = bm.WriteTo(buf) + panicOn(err) + return buf.Bytes() +} + // ImportRoaring is a low level interface for importing data to Pilosa when // extremely high throughput is desired. The data must be encoded in a // particular way which may be unintuitive (discussed below). The data is merged diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 2091cc850..c67962223 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -462,12 +462,13 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) * i++ } return &internal.ImportRoaringRequest{ - IndexCreatedAt: m.IndexCreatedAt, - FieldCreatedAt: m.FieldCreatedAt, - Clear: m.Clear, - Action: m.Action, - Block: uint64(m.Block), - Views: views, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Clear: m.Clear, + Action: m.Action, + Block: uint64(m.Block), + Views: views, + UpdateExistence: m.UpdateExistence, } } @@ -1256,6 +1257,7 @@ func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest m.Views = views m.IndexCreatedAt = pb.IndexCreatedAt m.FieldCreatedAt = pb.FieldCreatedAt + m.UpdateExistence = pb.UpdateExistence } func (s Serializer) decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) { diff --git a/handler.go b/handler.go index b50e6e946..ca5b119f5 100644 --- a/handler.go +++ b/handler.go @@ -243,12 +243,13 @@ const ( // ImportRoaringRequest describes the import request structure // for an import containing roaring-encoded data. type ImportRoaringRequest struct { - IndexCreatedAt int64 - FieldCreatedAt int64 - Clear bool - Action string // [set, clear, overwrite] - Block int - Views map[string][]byte + IndexCreatedAt int64 + FieldCreatedAt int64 + Clear bool + Action string // [set, clear, overwrite] + Block int + Views map[string][]byte + UpdateExistence bool } // ValidateWithTimestamp ensures that the payload of the request is valid. diff --git a/http/client_test.go b/http/client_test.go index b1b154647..87a010501 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -1435,3 +1435,47 @@ func TestClient_ServerInfoHasTxSrc(t *testing.T) { } pilosa.MustTxsrcToTxtype(si.TxSrc) // panics if invalid } +func TestClient_ImportRoaringExists(t *testing.T) { + cluster := test.MustNewCluster(t, 1) + err := cluster.Start() + if err != nil { + t.Fatalf("starting cluster: %v", err) + } + defer cluster.Close() + + node := cluster.GetNode(0) + _, err = node.API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = node.API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + // Send import request. + host := node.URL() + c := MustNewClient(host, http.GetHTTPClient(nil)) + // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] + roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") + + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { + t.Fatal(err) + } + qr, err := node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + got := qr.Results[0].(*pilosa.Row).Columns() + if !reflect.DeepEqual(got, []uint64{}) { + t.Fatalf(" Row unexpected columns: got %+v expected: %+v", got, []uint64{}) + } + roaringReq.UpdateExistence = true + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { + t.Fatal(err) + } + + expected := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537} + qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + got = qr.Results[0].(*pilosa.Row).Columns() + if !reflect.DeepEqual(got, expected) { + t.Fatalf("All unexpected columns: got %+v expected: %+v", got, expected) + } + +} diff --git a/internal/public.pb.go b/internal/public.pb.go index e406e06ab..109e68c08 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -2495,6 +2495,7 @@ type ImportRoaringRequest struct { Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` IndexCreatedAt int64 `protobuf:"varint,5,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` FieldCreatedAt int64 `protobuf:"varint,6,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` + UpdateExistence bool `protobuf:"varint,7,opt,name=UpdateExistence,proto3" json:"UpdateExistence,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2575,6 +2576,13 @@ func (m *ImportRoaringRequest) GetFieldCreatedAt() int64 { return 0 } +func (m *ImportRoaringRequest) GetUpdateExistence() bool { + if m != nil { + return m.UpdateExistence + } + return false +} + type ImportColumnAttrsRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Shard int64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` @@ -2761,117 +2769,119 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1757 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0xe4, 0x48, - 0x15, 0x8f, 0xdb, 0xee, 0x7f, 0xaf, 0x3b, 0x99, 0x6c, 0x4d, 0xcf, 0x62, 0x0d, 0x99, 0x6c, 0x63, - 0x05, 0xb6, 0x41, 0xab, 0xac, 0x12, 0x76, 0x60, 0x0e, 0xfc, 0xd9, 0x64, 0x3a, 0x4b, 0xac, 0x21, - 0xd9, 0xa1, 0x12, 0x65, 0xc5, 0x05, 0xc9, 0xe9, 0x2e, 0x7a, 0x2d, 0xdc, 0xed, 0xc6, 0xed, 0xde, - 0x4e, 0x2e, 0x48, 0x7c, 0x86, 0xbd, 0x70, 0x43, 0xdc, 0xf8, 0x1c, 0x5c, 0xe0, 0xc8, 0x11, 0x89, - 0x0b, 0x1a, 0xb8, 0xf2, 0x1d, 0xd0, 0x7b, 0xe5, 0x72, 0x95, 0xdd, 0x4e, 0x26, 0x1a, 0x71, 0xab, - 0xf7, 0xa7, 0x5e, 0xd5, 0xfb, 0xbd, 0x57, 0xef, 0x3d, 0x1b, 0xba, 0xf3, 0xe5, 0x75, 0x14, 0x8e, - 0xf6, 0xe7, 0x49, 0x9c, 0xc6, 0xac, 0x15, 0xce, 0x52, 0x91, 0xcc, 0x82, 0xc8, 0xfb, 0xa3, 0x05, - 0x36, 0x8f, 0x57, 0xcc, 0x85, 0xe6, 0xcb, 0x38, 0x5a, 0x4e, 0x67, 0x0b, 0xd7, 0xea, 0xdb, 0x03, - 0x87, 0x2b, 0x92, 0x31, 0x70, 0x5e, 0x89, 0xdb, 0x85, 0x6b, 0xf7, 0xed, 0x41, 0x9b, 0xd3, 0x9a, - 0xed, 0x41, 0xfd, 0x28, 0x4d, 0x93, 0x85, 0x5b, 0xeb, 0xdb, 0x83, 0xce, 0xe1, 0xd6, 0xbe, 0xb2, - 0xb7, 0x8f, 0x6c, 0x2e, 0x85, 0x68, 0x93, 0xc7, 0x41, 0x12, 0xce, 0x26, 0xae, 0xd3, 0xb7, 0x06, - 0x5d, 0xae, 0x48, 0xd6, 0x83, 0xba, 0x3f, 0x1b, 0x8b, 0x1b, 0xb7, 0xde, 0xb7, 0x06, 0x6d, 0x2e, - 0x09, 0xe4, 0x7e, 0x16, 0x8a, 0x68, 0xec, 0x36, 0x24, 0x97, 0x08, 0x6f, 0x1f, 0xda, 0x3c, 0x5e, - 0x9d, 0x05, 0x69, 0x12, 0xde, 0xb0, 0x6f, 0x81, 0xc3, 0xe3, 0x95, 0xbc, 0x63, 0xe7, 0x70, 0x53, - 0x9f, 0xcb, 0xe3, 0x15, 0x27, 0x91, 0x77, 0x06, 0xed, 0x8b, 0x70, 0x32, 0x13, 0x63, 0x74, 0xeb, - 0x03, 0xb0, 0x5f, 0xc7, 0xa8, 0x6e, 0xad, 0xab, 0xa3, 0x04, 0x15, 0xce, 0xc5, 0xc4, 0xad, 0x55, - 0x2a, 0x9c, 0x8b, 0x89, 0xf7, 0x02, 0xb6, 0x78, 0xbc, 0xf2, 0xc7, 0x62, 0x96, 0x86, 0xbf, 0x0e, - 0x45, 0x42, 0x80, 0xe4, 0x77, 0x70, 0xe4, 0xa1, 0x39, 0x48, 0x35, 0x0d, 0x92, 0xf7, 0x14, 0x1a, - 0xfe, 0xf0, 0xe7, 0xe1, 0x22, 0x65, 0xdb, 0x60, 0xfb, 0x43, 0xb5, 0x01, 0x97, 0x9e, 0x0f, 0xef, - 0x9d, 0xdc, 0xa4, 0x49, 0x30, 0x4a, 0xc5, 0xd8, 0x1f, 0x4a, 0xa8, 0xd9, 0x16, 0xd4, 0xfc, 0x21, - 0xdd, 0xd5, 0xe1, 0x35, 0x7f, 0xc8, 0xf6, 0xc0, 0xb9, 0x0a, 0x22, 0x05, 0xf2, 0xb6, 0xbe, 0x9c, - 0x34, 0xcb, 0x49, 0xea, 0x5d, 0x17, 0x4c, 0x65, 0x38, 0xbd, 0x0f, 0x0d, 0x42, 0x4f, 0x1e, 0xda, - 0xe6, 0x19, 0xc5, 0x9e, 0xeb, 0x30, 0x4b, 0xab, 0xdf, 0xd4, 0x56, 0xd7, 0x2e, 0x94, 0xe7, 0x80, - 0xf7, 0x0c, 0x9a, 0xaf, 0xc4, 0x2d, 0xf9, 0xa2, 0x3c, 0xb5, 0x0c, 0x4f, 0xff, 0x69, 0xc1, 0xe3, - 0x7c, 0xf7, 0x65, 0x70, 0x1d, 0x89, 0xab, 0x20, 0x5a, 0x0a, 0xb6, 0xa7, 0xfc, 0xb6, 0xaa, 0xee, - 0x7f, 0xba, 0x41, 0x58, 0xb0, 0x0f, 0x73, 0xec, 0x50, 0xed, 0x3d, 0xad, 0x96, 0x1d, 0x79, 0xba, - 0x91, 0x65, 0xdd, 0x0e, 0xb4, 0x8e, 0x2f, 0x7c, 0x32, 0xed, 0xda, 0x7d, 0x6b, 0x60, 0x9f, 0x6e, - 0xf0, 0x9c, 0xc3, 0x9e, 0x42, 0xf3, 0x6c, 0x99, 0x8a, 0x1b, 0x7f, 0x48, 0xd9, 0xe6, 0x9c, 0x6e, - 0x70, 0xc5, 0xc0, 0x9d, 0xb4, 0x7c, 0x25, 0x6e, 0x65, 0xca, 0xe1, 0x4e, 0xc5, 0x61, 0x3d, 0x70, - 0x8e, 0xe3, 0x38, 0xa2, 0xb4, 0x6b, 0xe1, 0x69, 0x48, 0x1d, 0x37, 0xa1, 0x4e, 0x86, 0xbd, 0xdf, - 0x41, 0xaf, 0xe8, 0x5c, 0x16, 0x2e, 0x06, 0x36, 0xda, 0xb3, 0x32, 0x7b, 0x48, 0xb0, 0x6d, 0x0a, - 0x61, 0x2d, 0x3b, 0x1f, 0x83, 0xf8, 0x1c, 0x1a, 0x64, 0x46, 0x3e, 0xa0, 0xce, 0xe1, 0xb3, 0x0a, - 0xc0, 0x35, 0x64, 0x3c, 0x53, 0x3e, 0x6e, 0x13, 0xe2, 0x9f, 0x27, 0xfe, 0xd0, 0xfb, 0x71, 0x19, - 0x5c, 0x8a, 0x25, 0x06, 0xe2, 0x3c, 0x98, 0x0a, 0x79, 0x3e, 0xa7, 0x35, 0xf2, 0x2e, 0x6f, 0xe7, - 0x82, 0x2e, 0xd0, 0xe6, 0xb4, 0xf6, 0x7e, 0x6f, 0xc1, 0x56, 0x71, 0x3f, 0xde, 0xc9, 0xc8, 0x8e, - 0x7b, 0xee, 0x44, 0x5a, 0x79, 0xf2, 0xbc, 0x28, 0x27, 0xcf, 0xee, 0x5d, 0xfb, 0xca, 0xf9, 0xf3, - 0x13, 0x70, 0x5e, 0x07, 0x61, 0xb2, 0x96, 0xe1, 0xdb, 0x12, 0x42, 0x9b, 0xae, 0x6b, 0xcb, 0x58, - 0xd4, 0x5f, 0xc6, 0xcb, 0x59, 0x2a, 0x31, 0xe4, 0x92, 0xf0, 0x4e, 0xa0, 0x8d, 0xfb, 0xa5, 0xe3, - 0x9e, 0x34, 0x96, 0xa5, 0x95, 0x51, 0x7b, 0x90, 0xcb, 0xe5, 0x41, 0x79, 0x29, 0xa9, 0x99, 0xa5, - 0xe4, 0x14, 0x00, 0xa5, 0x0b, 0x69, 0x67, 0x0f, 0xea, 0x44, 0x65, 0x20, 0x94, 0x0d, 0x49, 0xe1, - 0x1d, 0x96, 0x9e, 0x61, 0x01, 0x4b, 0x7f, 0xf0, 0x09, 0x8a, 0x65, 0x42, 0xe2, 0x6d, 0x6c, 0x9e, - 0xa5, 0xcc, 0x12, 0x5a, 0x12, 0xba, 0x78, 0xa5, 0x0d, 0x58, 0x86, 0x01, 0xe4, 0x62, 0x59, 0x19, - 0x2a, 0x3f, 0x89, 0xc0, 0x67, 0xcb, 0xe3, 0x95, 0x86, 0x24, 0xa3, 0xd8, 0xb7, 0xd5, 0x29, 0x0e, - 0xf9, 0xfc, 0xc8, 0x78, 0x4a, 0x78, 0x0b, 0x75, 0xec, 0xaf, 0x00, 0x7e, 0x96, 0xc4, 0xcb, 0x39, - 0x81, 0xc6, 0x06, 0x50, 0x27, 0x2a, 0xf3, 0x8f, 0xe9, 0x4d, 0xea, 0x6e, 0x5c, 0x2a, 0x54, 0x83, - 0x8e, 0xc1, 0x39, 0x9a, 0x4c, 0xe4, 0x4b, 0xe3, 0xb8, 0xc4, 0x54, 0x6a, 0x5d, 0x05, 0x51, 0x2e, - 0xbe, 0x0a, 0xa2, 0xcc, 0x6f, 0x5c, 0x16, 0xcd, 0xd8, 0xca, 0xcc, 0x53, 0x68, 0x7d, 0x16, 0xc5, - 0x41, 0x8a, 0xca, 0x68, 0xcb, 0xe2, 0x39, 0xcd, 0x0e, 0x00, 0x86, 0x62, 0x14, 0x4e, 0x83, 0x08, - 0xa5, 0x4e, 0xb9, 0x00, 0x64, 0x32, 0x6e, 0x28, 0x79, 0xcf, 0xa1, 0x99, 0x51, 0xd5, 0xd8, 0x23, - 0xf7, 0x62, 0x14, 0x44, 0x42, 0xdd, 0x82, 0x08, 0xef, 0x0b, 0xd8, 0x94, 0xc9, 0x88, 0xad, 0xe9, - 0x42, 0xa4, 0x0f, 0x48, 0xc5, 0x07, 0x35, 0x39, 0xef, 0xcf, 0x16, 0x38, 0xb8, 0x52, 0x06, 0x2c, - 0x6d, 0xc0, 0x7c, 0x8d, 0x8e, 0x7c, 0x8d, 0xac, 0x0f, 0x9d, 0x8b, 0x14, 0x7b, 0xa0, 0x2e, 0x63, - 0x6d, 0x6e, 0xb2, 0x10, 0x2f, 0x7f, 0x96, 0xea, 0x70, 0xdb, 0x3c, 0xa7, 0xd9, 0x0e, 0xb4, 0xb1, - 0x36, 0x49, 0x21, 0x16, 0xb2, 0x16, 0xd7, 0x0c, 0xb6, 0x0b, 0xa0, 0x90, 0x5d, 0x0a, 0xaa, 0x66, - 0x16, 0x37, 0x38, 0xde, 0xc7, 0xd0, 0xc4, 0x9b, 0x9e, 0x05, 0x73, 0xed, 0x9b, 0x75, 0x9f, 0x6f, - 0x7f, 0xaa, 0x41, 0xf7, 0x17, 0x4b, 0x91, 0xdc, 0x72, 0xf1, 0xdb, 0xa5, 0x58, 0xa4, 0x88, 0x2d, - 0xd1, 0x2a, 0x97, 0x89, 0xc0, 0xac, 0xbd, 0xf8, 0x32, 0x48, 0xc6, 0x12, 0x29, 0x87, 0x67, 0x14, - 0xfa, 0xaa, 0x31, 0x5f, 0x90, 0xaf, 0x2d, 0x6e, 0xb2, 0x28, 0xdf, 0xc5, 0x34, 0x4e, 0x95, 0x33, - 0x19, 0xc5, 0x06, 0xf0, 0xe8, 0xe4, 0x66, 0x14, 0x2d, 0xc7, 0x82, 0xc7, 0x2b, 0xb9, 0x9b, 0x8a, - 0x33, 0x2f, 0xb3, 0xd9, 0x77, 0xb0, 0xb8, 0x11, 0x4b, 0x95, 0xa6, 0x26, 0x29, 0x96, 0xb8, 0xec, - 0x00, 0xba, 0x27, 0xd3, 0x6b, 0x31, 0x1e, 0x8b, 0xf1, 0x30, 0x48, 0x03, 0xb7, 0x55, 0x35, 0x40, - 0x14, 0x54, 0xd8, 0x1e, 0x6c, 0xbe, 0x4e, 0xc4, 0x65, 0x12, 0xcc, 0x16, 0x51, 0x90, 0x8a, 0xb1, - 0xdb, 0x26, 0xcb, 0x45, 0xa6, 0xf7, 0xb5, 0x05, 0x9b, 0x19, 0x46, 0x8b, 0x79, 0x3c, 0x5b, 0x08, - 0x4c, 0x84, 0x93, 0x24, 0x51, 0x89, 0x70, 0x92, 0x24, 0xec, 0x63, 0x68, 0x72, 0xb1, 0x58, 0x46, - 0xa9, 0xca, 0xa5, 0x27, 0xfa, 0x5c, 0xb5, 0x77, 0x19, 0xa5, 0x5c, 0x69, 0xb1, 0x9f, 0xc2, 0x56, - 0x21, 0x5b, 0x55, 0xf3, 0xf8, 0x86, 0xde, 0x57, 0x90, 0xf3, 0x92, 0xba, 0xf7, 0xdf, 0x3a, 0x74, - 0x0c, 0xcb, 0x79, 0x2a, 0x22, 0x8a, 0x9b, 0x59, 0x2a, 0x7e, 0x40, 0x93, 0xdf, 0x1d, 0xb3, 0x11, - 0x56, 0xae, 0x2e, 0x58, 0xe7, 0x59, 0xf2, 0x5a, 0xe7, 0xba, 0x5c, 0xda, 0xf7, 0x95, 0x4b, 0x9c, - 0x23, 0xbf, 0x0c, 0x66, 0x13, 0x31, 0xa6, 0xe4, 0x6d, 0x71, 0x45, 0xb2, 0x7d, 0x5d, 0x3b, 0x28, - 0xda, 0x85, 0x8a, 0xa4, 0x24, 0x5c, 0xd7, 0x17, 0x59, 0x0b, 0x71, 0x7e, 0x68, 0xca, 0xac, 0x92, - 0x14, 0xfb, 0x11, 0x6c, 0x7d, 0x1e, 0x8d, 0x75, 0x9d, 0x5b, 0x64, 0xb1, 0xec, 0x69, 0x6b, 0x5a, - 0xc8, 0x4b, 0xba, 0xec, 0xd3, 0xf2, 0x38, 0x47, 0x51, 0xed, 0x1c, 0xba, 0x05, 0xff, 0x0d, 0x39, - 0x2f, 0x8f, 0x7f, 0x07, 0xc6, 0x7c, 0xe9, 0x02, 0x6d, 0x7e, 0xac, 0x37, 0xe7, 0x22, 0x6e, 0x4c, - 0xa1, 0x9f, 0x98, 0x7d, 0xc7, 0xed, 0xd0, 0x9e, 0x5e, 0x11, 0x3f, 0x29, 0xe3, 0x66, 0x7f, 0x3a, - 0x30, 0x9a, 0x9e, 0xdb, 0x2d, 0x1f, 0x94, 0x8b, 0xb8, 0xd1, 0x1a, 0xfd, 0x8a, 0x59, 0xd0, 0xdd, - 0xa4, 0xad, 0xd5, 0x83, 0x9e, 0x54, 0xe1, 0x15, 0x13, 0xe4, 0xa7, 0xe5, 0xa9, 0xc1, 0xdd, 0x2a, - 0x03, 0x55, 0x94, 0xf3, 0xf2, 0x94, 0x71, 0x60, 0x0c, 0xee, 0xee, 0xa3, 0xf2, 0xfd, 0x73, 0x11, - 0x37, 0xc6, 0xfb, 0x1f, 0x42, 0xc7, 0x0c, 0xec, 0x36, 0x6d, 0x7a, 0x52, 0x15, 0xd8, 0x05, 0x37, - 0x35, 0xbd, 0xbf, 0xd6, 0x60, 0xd3, 0x9f, 0xce, 0xe3, 0x24, 0x35, 0x4a, 0x95, 0xfc, 0xc4, 0xb0, - 0x2a, 0x3f, 0x31, 0x6a, 0xa5, 0x66, 0x4c, 0x25, 0x8b, 0x4a, 0x94, 0xc3, 0x25, 0x61, 0x24, 0xa0, - 0x53, 0x48, 0xc0, 0x1d, 0x68, 0xcb, 0xd7, 0x86, 0xa2, 0x3a, 0x89, 0x34, 0x43, 0x7e, 0xf4, 0xac, - 0x68, 0xa0, 0x6d, 0xd2, 0x88, 0xac, 0x48, 0x2c, 0xcf, 0x52, 0x8d, 0x84, 0x2d, 0x12, 0x1a, 0x1c, - 0x94, 0x5f, 0x86, 0x53, 0xb1, 0x48, 0x83, 0xe9, 0x1c, 0xeb, 0x9d, 0x3d, 0xb0, 0xb9, 0xc1, 0xc1, - 0x52, 0x47, 0x4e, 0xbc, 0x4c, 0x04, 0x56, 0x9e, 0xa3, 0x94, 0x52, 0xd7, 0xe6, 0x25, 0x2e, 0xea, - 0x91, 0x5b, 0x5a, 0x0f, 0xa4, 0x5e, 0x91, 0x4b, 0xed, 0x3a, 0x12, 0x41, 0x42, 0x09, 0xd9, 0xe2, - 0x92, 0xf0, 0xfe, 0x51, 0x03, 0x26, 0x91, 0x94, 0x03, 0xe9, 0xff, 0x0d, 0xce, 0xfb, 0x61, 0x2b, - 0x82, 0xd3, 0x5c, 0x03, 0xe7, 0xfd, 0x7c, 0x8c, 0x96, 0xc0, 0x64, 0x14, 0xf6, 0x18, 0xdd, 0xe1, - 0x24, 0xaa, 0x16, 0x37, 0x59, 0xcc, 0x83, 0xae, 0xd1, 0x5e, 0xf1, 0xbd, 0xa3, 0xed, 0x02, 0xaf, - 0x02, 0x5a, 0x78, 0x20, 0xb4, 0x9d, 0xfb, 0xa1, 0xed, 0x9a, 0xd0, 0x7e, 0x6d, 0x41, 0xf7, 0x28, - 0x8d, 0xa7, 0xe1, 0x88, 0x8b, 0x51, 0x9c, 0x8c, 0xef, 0x06, 0x55, 0xc2, 0x57, 0x33, 0xe1, 0xdb, - 0x07, 0xdb, 0xff, 0x2a, 0xc9, 0x8a, 0xef, 0x8e, 0x31, 0x00, 0xae, 0xc5, 0x8a, 0xa3, 0x22, 0xfb, - 0x10, 0x6a, 0x7e, 0x42, 0x99, 0x5b, 0x68, 0x1b, 0x85, 0x47, 0xc2, 0x6b, 0x7e, 0xe2, 0x7d, 0x04, - 0x3d, 0x79, 0x29, 0x25, 0xca, 0xda, 0x58, 0x0f, 0xea, 0x27, 0x49, 0x12, 0xab, 0x46, 0x26, 0x09, - 0xef, 0x06, 0x7a, 0x79, 0xf3, 0xc3, 0xc0, 0xbc, 0x4b, 0x7e, 0x54, 0xfd, 0x51, 0xe8, 0x43, 0xe7, - 0x3c, 0x4e, 0xbf, 0x48, 0xc2, 0x94, 0x6a, 0x8d, 0xec, 0x1d, 0x26, 0xcb, 0xfb, 0x2e, 0x3c, 0x29, - 0x9d, 0xac, 0xfb, 0x2d, 0xa6, 0x94, 0xad, 0xbf, 0xae, 0x2f, 0xe0, 0x71, 0xae, 0xea, 0x0f, 0xdf, - 0xe9, 0x8e, 0xeb, 0x46, 0xbf, 0x67, 0x78, 0x4e, 0x46, 0xb3, 0xe3, 0x2b, 0xbc, 0xf1, 0x8e, 0xc1, - 0xcd, 0xd0, 0x94, 0x3f, 0x3c, 0xb2, 0x1b, 0x5c, 0x85, 0x62, 0x75, 0xd7, 0x77, 0x1b, 0x4d, 0x25, - 0x35, 0xfa, 0x4d, 0x42, 0x6b, 0xef, 0x3f, 0x16, 0xf4, 0xaa, 0x8c, 0xe8, 0xe4, 0xb2, 0x8c, 0xe4, - 0x62, 0x2f, 0xa0, 0xfe, 0x55, 0x28, 0x56, 0x6a, 0xc2, 0xf0, 0xd6, 0x42, 0xbe, 0x76, 0x13, 0x2e, - 0x37, 0xe0, 0xd3, 0x3a, 0x1a, 0xa5, 0x61, 0x3c, 0x53, 0x1f, 0x1d, 0x92, 0xc2, 0x73, 0x8e, 0xa3, - 0x78, 0xf4, 0x1b, 0xf9, 0x39, 0xcd, 0x25, 0x51, 0xf1, 0x54, 0xea, 0x0f, 0x7c, 0x2a, 0x8d, 0xaa, - 0xa7, 0xe2, 0xfd, 0xc5, 0x52, 0x58, 0x19, 0x83, 0xe1, 0x5b, 0x23, 0xa6, 0x1f, 0x88, 0xad, 0x1e, - 0x88, 0x2b, 0xa7, 0x5b, 0x3d, 0xc4, 0x2b, 0x12, 0x27, 0x6a, 0x5c, 0xd2, 0xbf, 0x14, 0x87, 0xa2, - 0x94, 0xd3, 0x6f, 0xa9, 0x4a, 0xeb, 0xce, 0x36, 0xaa, 0x9c, 0xf5, 0x7e, 0x59, 0xe8, 0x5b, 0x68, - 0xf4, 0x68, 0x32, 0x49, 0xc4, 0x24, 0x48, 0x55, 0x9c, 0x35, 0x83, 0x7d, 0x04, 0x0d, 0x52, 0x56, - 0xa1, 0xaa, 0x1e, 0x5c, 0x32, 0x9d, 0xe3, 0xed, 0xbf, 0xbd, 0xd9, 0xb5, 0xfe, 0xfe, 0x66, 0xd7, - 0xfa, 0xd7, 0x9b, 0x5d, 0xeb, 0x0f, 0xff, 0xde, 0xdd, 0xb8, 0x6e, 0xd0, 0x3f, 0xbc, 0xef, 0xff, - 0x2f, 0x00, 0x00, 0xff, 0xff, 0x0a, 0x79, 0xf3, 0x2b, 0xd3, 0x13, 0x00, 0x00, + // 1779 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0x23, 0x49, + 0x15, 0x4f, 0xbb, 0xdb, 0xb1, 0xfd, 0xec, 0x64, 0xb2, 0x35, 0x9e, 0xa5, 0x35, 0xcc, 0x64, 0x43, + 0x2b, 0xb0, 0x06, 0xad, 0xb2, 0xca, 0xb0, 0x03, 0x73, 0xe0, 0xcf, 0x26, 0xe3, 0x2c, 0x69, 0x0d, + 0x93, 0x1d, 0x2a, 0x43, 0x56, 0x5c, 0x90, 0x3a, 0x76, 0xe1, 0x6d, 0xd1, 0x76, 0x9b, 0x76, 0x79, + 0x9d, 0x5c, 0x90, 0xf8, 0x02, 0x5c, 0xf6, 0xc2, 0x0d, 0x71, 0xe3, 0x73, 0x70, 0x81, 0x23, 0x47, + 0x24, 0x2e, 0x68, 0x38, 0xf3, 0x1d, 0xd0, 0x7b, 0xd5, 0xd5, 0x55, 0xdd, 0xee, 0x64, 0xa3, 0xd1, + 0xde, 0xea, 0xfd, 0xa9, 0x57, 0xf5, 0x7e, 0xef, 0xd5, 0x7b, 0xaf, 0x1b, 0x7a, 0xf3, 0xe5, 0x65, + 0x12, 0x8f, 0x0e, 0xe6, 0x59, 0x2a, 0x53, 0xd6, 0x8e, 0x67, 0x52, 0x64, 0xb3, 0x28, 0x09, 0xfe, + 0xec, 0x80, 0xcb, 0xd3, 0x15, 0xf3, 0xa1, 0xf5, 0x3c, 0x4d, 0x96, 0xd3, 0xd9, 0xc2, 0x77, 0xf6, + 0xdc, 0x81, 0xc7, 0x35, 0xc9, 0x18, 0x78, 0x2f, 0xc4, 0xf5, 0xc2, 0x77, 0xf7, 0xdc, 0x41, 0x87, + 0xd3, 0x9a, 0xed, 0x43, 0xf3, 0x48, 0xca, 0x6c, 0xe1, 0x37, 0xf6, 0xdc, 0x41, 0xf7, 0xc9, 0xf6, + 0x81, 0xb6, 0x77, 0x80, 0x6c, 0xae, 0x84, 0x68, 0x93, 0xa7, 0x51, 0x16, 0xcf, 0x26, 0xbe, 0xb7, + 0xe7, 0x0c, 0x7a, 0x5c, 0x93, 0xac, 0x0f, 0xcd, 0x70, 0x36, 0x16, 0x57, 0x7e, 0x73, 0xcf, 0x19, + 0x74, 0xb8, 0x22, 0x90, 0xfb, 0x49, 0x2c, 0x92, 0xb1, 0xbf, 0xa9, 0xb8, 0x44, 0x04, 0x07, 0xd0, + 0xe1, 0xe9, 0xea, 0x65, 0x24, 0xb3, 0xf8, 0x8a, 0x7d, 0x0b, 0x3c, 0x9e, 0xae, 0xd4, 0x1d, 0xbb, + 0x4f, 0xb6, 0xcc, 0xb9, 0x3c, 0x5d, 0x71, 0x12, 0x05, 0x2f, 0xa1, 0x73, 0x1e, 0x4f, 0x66, 0x62, + 0x8c, 0x6e, 0xbd, 0x07, 0xee, 0xab, 0x14, 0xd5, 0x9d, 0x75, 0x75, 0x94, 0xa0, 0xc2, 0x99, 0x98, + 0xf8, 0x8d, 0x5a, 0x85, 0x33, 0x31, 0x09, 0x9e, 0xc1, 0x36, 0x4f, 0x57, 0xe1, 0x58, 0xcc, 0x64, + 0xfc, 0x9b, 0x58, 0x64, 0x04, 0x48, 0x71, 0x07, 0x4f, 0x1d, 0x5a, 0x80, 0xd4, 0x30, 0x20, 0x05, + 0x0f, 0x61, 0x33, 0x1c, 0xfe, 0x3c, 0x5e, 0x48, 0xb6, 0x03, 0x6e, 0x38, 0xd4, 0x1b, 0x70, 0x19, + 0x84, 0xf0, 0xce, 0xc9, 0x95, 0xcc, 0xa2, 0x91, 0x14, 0xe3, 0x70, 0xa8, 0xa0, 0x66, 0xdb, 0xd0, + 0x08, 0x87, 0x74, 0x57, 0x8f, 0x37, 0xc2, 0x21, 0xdb, 0x07, 0xef, 0x22, 0x4a, 0x34, 0xc8, 0x3b, + 0xe6, 0x72, 0xca, 0x2c, 0x27, 0x69, 0x70, 0x59, 0x32, 0x95, 0xe3, 0xf4, 0x2e, 0x6c, 0x12, 0x7a, + 0xea, 0xd0, 0x0e, 0xcf, 0x29, 0xf6, 0xd4, 0x84, 0x59, 0x59, 0xfd, 0xa6, 0xb1, 0xba, 0x76, 0xa1, + 0x22, 0x07, 0x82, 0xc7, 0xd0, 0x7a, 0x21, 0xae, 0xc9, 0x17, 0xed, 0xa9, 0x63, 0x79, 0xfa, 0x6f, + 0x07, 0xee, 0x17, 0xbb, 0x5f, 0x47, 0x97, 0x89, 0xb8, 0x88, 0x92, 0xa5, 0x60, 0xfb, 0xda, 0x6f, + 0xa7, 0xee, 0xfe, 0xa7, 0x1b, 0x84, 0x05, 0x7b, 0xbf, 0xc0, 0x0e, 0xd5, 0xde, 0x31, 0x6a, 0xf9, + 0x91, 0xa7, 0x1b, 0x79, 0xd6, 0x3d, 0x82, 0xf6, 0xf1, 0x79, 0x48, 0xa6, 0x7d, 0x77, 0xcf, 0x19, + 0xb8, 0xa7, 0x1b, 0xbc, 0xe0, 0xb0, 0x87, 0xd0, 0x7a, 0xb9, 0x94, 0xe2, 0x2a, 0x1c, 0x52, 0xb6, + 0x79, 0xa7, 0x1b, 0x5c, 0x33, 0x70, 0x27, 0x2d, 0x5f, 0x88, 0x6b, 0x95, 0x72, 0xb8, 0x53, 0x73, + 0x58, 0x1f, 0xbc, 0xe3, 0x34, 0x4d, 0x28, 0xed, 0xda, 0x78, 0x1a, 0x52, 0xc7, 0x2d, 0x68, 0x92, + 0xe1, 0xe0, 0xf7, 0xd0, 0x2f, 0x3b, 0x97, 0x87, 0x8b, 0x81, 0x8b, 0xf6, 0x9c, 0xdc, 0x1e, 0x12, + 0x6c, 0x87, 0x42, 0xd8, 0xc8, 0xcf, 0xc7, 0x20, 0x3e, 0x85, 0x4d, 0x32, 0xa3, 0x1e, 0x50, 0xf7, + 0xc9, 0xe3, 0x1a, 0xc0, 0x0d, 0x64, 0x3c, 0x57, 0x3e, 0xee, 0x10, 0xe2, 0x9f, 0x66, 0xe1, 0x30, + 0xf8, 0x71, 0x15, 0x5c, 0x8a, 0x25, 0x06, 0xe2, 0x2c, 0x9a, 0x0a, 0x75, 0x3e, 0xa7, 0x35, 0xf2, + 0x5e, 0x5f, 0xcf, 0x05, 0x5d, 0xa0, 0xc3, 0x69, 0x1d, 0xfc, 0xc1, 0x81, 0xed, 0xf2, 0x7e, 0xbc, + 0x93, 0x95, 0x1d, 0xb7, 0xdc, 0x89, 0xb4, 0x8a, 0xe4, 0x79, 0x56, 0x4d, 0x9e, 0xdd, 0x9b, 0xf6, + 0x55, 0xf3, 0xe7, 0x27, 0xe0, 0xbd, 0x8a, 0xe2, 0x6c, 0x2d, 0xc3, 0x77, 0x14, 0x84, 0x2e, 0x5d, + 0xd7, 0x55, 0xb1, 0x68, 0x3e, 0x4f, 0x97, 0x33, 0xa9, 0x30, 0xe4, 0x8a, 0x08, 0x4e, 0xa0, 0x83, + 0xfb, 0x95, 0xe3, 0x81, 0x32, 0x96, 0xa7, 0x95, 0x55, 0x7b, 0x90, 0xcb, 0xd5, 0x41, 0x45, 0x29, + 0x69, 0xd8, 0xa5, 0xe4, 0x14, 0x00, 0xa5, 0x0b, 0x65, 0x67, 0x1f, 0x9a, 0x44, 0xe5, 0x20, 0x54, + 0x0d, 0x29, 0xe1, 0x0d, 0x96, 0x1e, 0x63, 0x01, 0x93, 0x3f, 0xf8, 0x08, 0xc5, 0x2a, 0x21, 0xf1, + 0x36, 0x2e, 0xcf, 0x53, 0x66, 0x09, 0x6d, 0x05, 0x5d, 0xba, 0x32, 0x06, 0x1c, 0xcb, 0x00, 0x72, + 0xb1, 0xac, 0x0c, 0xb5, 0x9f, 0x44, 0xe0, 0xb3, 0xe5, 0xe9, 0xca, 0x40, 0x92, 0x53, 0xec, 0xdb, + 0xfa, 0x14, 0x8f, 0x7c, 0xbe, 0x67, 0x3d, 0x25, 0xbc, 0x85, 0x3e, 0xf6, 0xd7, 0x00, 0x3f, 0xcb, + 0xd2, 0xe5, 0x9c, 0x40, 0x63, 0x03, 0x68, 0x12, 0x95, 0xfb, 0xc7, 0xcc, 0x26, 0x7d, 0x37, 0xae, + 0x14, 0xea, 0x41, 0xc7, 0xe0, 0x1c, 0x4d, 0x26, 0xea, 0xa5, 0x71, 0x5c, 0x62, 0x2a, 0xb5, 0x2f, + 0xa2, 0xa4, 0x10, 0x5f, 0x44, 0x49, 0xee, 0x37, 0x2e, 0xcb, 0x66, 0x5c, 0x6d, 0xe6, 0x21, 0xb4, + 0x3f, 0x49, 0xd2, 0x48, 0xa2, 0x32, 0xda, 0x72, 0x78, 0x41, 0xb3, 0x43, 0x80, 0xa1, 0x18, 0xc5, + 0xd3, 0x28, 0x41, 0xa9, 0x57, 0x2d, 0x00, 0xb9, 0x8c, 0x5b, 0x4a, 0xc1, 0x53, 0x68, 0xe5, 0x54, + 0x3d, 0xf6, 0xc8, 0x3d, 0x1f, 0x45, 0x89, 0xd0, 0xb7, 0x20, 0x22, 0xf8, 0x0c, 0xb6, 0x54, 0x32, + 0x62, 0x6b, 0x3a, 0x17, 0xf2, 0x0e, 0xa9, 0x78, 0xa7, 0x26, 0x17, 0xfc, 0xd5, 0x01, 0x0f, 0x57, + 0xda, 0x80, 0x63, 0x0c, 0xd8, 0xaf, 0xd1, 0x53, 0xaf, 0x91, 0xed, 0x41, 0xf7, 0x5c, 0x62, 0x0f, + 0x34, 0x65, 0xac, 0xc3, 0x6d, 0x16, 0xe2, 0x15, 0xce, 0xa4, 0x09, 0xb7, 0xcb, 0x0b, 0x9a, 0x3d, + 0x82, 0x0e, 0xd6, 0x26, 0x25, 0xc4, 0x42, 0xd6, 0xe6, 0x86, 0xc1, 0x76, 0x01, 0x34, 0xb2, 0x4b, + 0x41, 0xd5, 0xcc, 0xe1, 0x16, 0x27, 0xf8, 0x10, 0x5a, 0x78, 0xd3, 0x97, 0xd1, 0xdc, 0xf8, 0xe6, + 0xdc, 0xe6, 0xdb, 0x5f, 0x1a, 0xd0, 0xfb, 0xc5, 0x52, 0x64, 0xd7, 0x5c, 0xfc, 0x6e, 0x29, 0x16, + 0x12, 0xb1, 0x25, 0x5a, 0xe7, 0x32, 0x11, 0x98, 0xb5, 0xe7, 0x9f, 0x47, 0xd9, 0x58, 0x21, 0xe5, + 0xf1, 0x9c, 0x42, 0x5f, 0x0d, 0xe6, 0x0b, 0xf2, 0xb5, 0xcd, 0x6d, 0x16, 0xe5, 0xbb, 0x98, 0xa6, + 0x52, 0x3b, 0x93, 0x53, 0x6c, 0x00, 0xf7, 0x4e, 0xae, 0x46, 0xc9, 0x72, 0x2c, 0x78, 0xba, 0x52, + 0xbb, 0xa9, 0x38, 0xf3, 0x2a, 0x9b, 0x7d, 0x07, 0x8b, 0x1b, 0xb1, 0x74, 0x69, 0x6a, 0x91, 0x62, + 0x85, 0xcb, 0x0e, 0xa1, 0x77, 0x32, 0xbd, 0x14, 0xe3, 0xb1, 0x18, 0x0f, 0x23, 0x19, 0xf9, 0xed, + 0xba, 0x01, 0xa2, 0xa4, 0xc2, 0xf6, 0x61, 0xeb, 0x55, 0x26, 0x5e, 0x67, 0xd1, 0x6c, 0x91, 0x44, + 0x52, 0x8c, 0xfd, 0x0e, 0x59, 0x2e, 0x33, 0x83, 0x2f, 0x1d, 0xd8, 0xca, 0x31, 0x5a, 0xcc, 0xd3, + 0xd9, 0x42, 0x60, 0x22, 0x9c, 0x64, 0x99, 0x4e, 0x84, 0x93, 0x2c, 0x63, 0x1f, 0x42, 0x8b, 0x8b, + 0xc5, 0x32, 0x91, 0x3a, 0x97, 0x1e, 0x98, 0x73, 0xf5, 0xde, 0x65, 0x22, 0xb9, 0xd6, 0x62, 0x3f, + 0x85, 0xed, 0x52, 0xb6, 0xea, 0xe6, 0xf1, 0x0d, 0xb3, 0xaf, 0x24, 0xe7, 0x15, 0xf5, 0xe0, 0x7f, + 0x4d, 0xe8, 0x5a, 0x96, 0x8b, 0x54, 0x44, 0x14, 0xb7, 0xf2, 0x54, 0x7c, 0x8f, 0x26, 0xbf, 0x1b, + 0x66, 0x23, 0xac, 0x5c, 0x3d, 0x70, 0xce, 0xf2, 0xe4, 0x75, 0xce, 0x4c, 0xb9, 0x74, 0x6f, 0x2b, + 0x97, 0x38, 0x47, 0x7e, 0x1e, 0xcd, 0x26, 0x62, 0x4c, 0xc9, 0xdb, 0xe6, 0x9a, 0x64, 0x07, 0xa6, + 0x76, 0x50, 0xb4, 0x4b, 0x15, 0x49, 0x4b, 0xb8, 0xa9, 0x2f, 0xaa, 0x16, 0xe2, 0xfc, 0xd0, 0x52, + 0x59, 0xa5, 0x28, 0xf6, 0x23, 0xd8, 0xfe, 0x34, 0x19, 0x9b, 0x3a, 0xb7, 0xc8, 0x63, 0xd9, 0x37, + 0xd6, 0x8c, 0x90, 0x57, 0x74, 0xd9, 0xc7, 0xd5, 0x71, 0x8e, 0xa2, 0xda, 0x7d, 0xe2, 0x97, 0xfc, + 0xb7, 0xe4, 0xbc, 0x3a, 0xfe, 0x1d, 0x5a, 0xf3, 0xa5, 0x0f, 0xb4, 0xf9, 0xbe, 0xd9, 0x5c, 0x88, + 0xb8, 0x35, 0x85, 0x7e, 0x64, 0xf7, 0x1d, 0xbf, 0x4b, 0x7b, 0xfa, 0x65, 0xfc, 0x94, 0x8c, 0xdb, + 0xfd, 0xe9, 0xd0, 0x6a, 0x7a, 0x7e, 0xaf, 0x7a, 0x50, 0x21, 0xe2, 0x56, 0x6b, 0x0c, 0x6b, 0x66, + 0x41, 0x7f, 0x8b, 0xb6, 0xd6, 0x0f, 0x7a, 0x4a, 0x85, 0xd7, 0x4c, 0x90, 0x1f, 0x57, 0xa7, 0x06, + 0x7f, 0xbb, 0x0a, 0x54, 0x59, 0xce, 0xab, 0x53, 0xc6, 0xa1, 0x35, 0xb8, 0xfb, 0xf7, 0xaa, 0xf7, + 0x2f, 0x44, 0xdc, 0x1a, 0xef, 0x7f, 0x08, 0x5d, 0x3b, 0xb0, 0x3b, 0xb4, 0xe9, 0x41, 0x5d, 0x60, + 0x17, 0xdc, 0xd6, 0x0c, 0xfe, 0xde, 0x80, 0xad, 0x70, 0x3a, 0x4f, 0x33, 0x69, 0x95, 0x2a, 0xf5, + 0x89, 0xe1, 0xd4, 0x7e, 0x62, 0x34, 0x2a, 0xcd, 0x98, 0x4a, 0x16, 0x95, 0x28, 0x8f, 0x2b, 0xc2, + 0x4a, 0x40, 0xaf, 0x94, 0x80, 0x8f, 0xa0, 0xa3, 0x5e, 0x1b, 0x8a, 0x9a, 0x24, 0x32, 0x0c, 0xf5, + 0xd1, 0xb3, 0xa2, 0x81, 0xb6, 0x45, 0x23, 0xb2, 0x26, 0xb1, 0x3c, 0x2b, 0x35, 0x12, 0xb6, 0x49, + 0x68, 0x71, 0x50, 0xfe, 0x3a, 0x9e, 0x8a, 0x85, 0x8c, 0xa6, 0x73, 0xac, 0x77, 0xee, 0xc0, 0xe5, + 0x16, 0x07, 0x4b, 0x1d, 0x39, 0xf1, 0x3c, 0x13, 0x58, 0x79, 0x8e, 0x24, 0xa5, 0xae, 0xcb, 0x2b, + 0x5c, 0xd4, 0x23, 0xb7, 0x8c, 0x1e, 0x28, 0xbd, 0x32, 0x97, 0xda, 0x75, 0x22, 0xa2, 0x8c, 0x12, + 0xb2, 0xcd, 0x15, 0x11, 0xfc, 0xab, 0x01, 0x4c, 0x21, 0xa9, 0x06, 0xd2, 0xaf, 0x0d, 0xce, 0xdb, + 0x61, 0x2b, 0x83, 0xd3, 0x5a, 0x03, 0xe7, 0xdd, 0x62, 0x8c, 0x56, 0xc0, 0xe4, 0x14, 0xf6, 0x18, + 0xd3, 0xe1, 0x14, 0xaa, 0x0e, 0xb7, 0x59, 0x2c, 0x80, 0x9e, 0xd5, 0x5e, 0xf1, 0xbd, 0xa3, 0xed, + 0x12, 0xaf, 0x06, 0x5a, 0xb8, 0x23, 0xb4, 0xdd, 0xdb, 0xa1, 0xed, 0xd9, 0xd0, 0x7e, 0xe9, 0x40, + 0xef, 0x48, 0xa6, 0xd3, 0x78, 0xc4, 0xc5, 0x28, 0xcd, 0xc6, 0x37, 0x83, 0xaa, 0xe0, 0x6b, 0xd8, + 0xf0, 0x1d, 0x80, 0x1b, 0x7e, 0x91, 0xe5, 0xc5, 0xf7, 0x91, 0x35, 0x00, 0xae, 0xc5, 0x8a, 0xa3, + 0x22, 0x7b, 0x1f, 0x1a, 0x61, 0x46, 0x99, 0x5b, 0x6a, 0x1b, 0xa5, 0x47, 0xc2, 0x1b, 0x61, 0x16, + 0x7c, 0x00, 0x7d, 0x75, 0x29, 0x2d, 0xca, 0xdb, 0x58, 0x1f, 0x9a, 0x27, 0x59, 0x96, 0xea, 0x46, + 0xa6, 0x88, 0xe0, 0x0a, 0xfa, 0x45, 0xf3, 0xc3, 0xc0, 0xbc, 0x4d, 0x7e, 0xd4, 0xfd, 0x51, 0xd8, + 0x83, 0xee, 0x59, 0x2a, 0x3f, 0xcb, 0x62, 0x49, 0xb5, 0x46, 0xf5, 0x0e, 0x9b, 0x15, 0x7c, 0x17, + 0x1e, 0x54, 0x4e, 0x36, 0xfd, 0x16, 0x53, 0xca, 0x35, 0x5f, 0xd7, 0xe7, 0x70, 0xbf, 0x50, 0x0d, + 0x87, 0x6f, 0x75, 0xc7, 0x75, 0xa3, 0xdf, 0xb3, 0x3c, 0x27, 0xa3, 0xf9, 0xf1, 0x35, 0xde, 0x04, + 0xc7, 0xe0, 0xe7, 0x68, 0xaa, 0x1f, 0x1e, 0xf9, 0x0d, 0x2e, 0x62, 0xb1, 0xba, 0xe9, 0xbb, 0x8d, + 0xa6, 0x92, 0x06, 0xfd, 0x26, 0xa1, 0x75, 0xf0, 0xc7, 0x06, 0xf4, 0xeb, 0x8c, 0x98, 0xe4, 0x72, + 0xac, 0xe4, 0x62, 0xcf, 0xa0, 0xf9, 0x45, 0x2c, 0x56, 0x7a, 0xc2, 0x08, 0xd6, 0x42, 0xbe, 0x76, + 0x13, 0xae, 0x36, 0xe0, 0xd3, 0x3a, 0x1a, 0xc9, 0x38, 0x9d, 0xe9, 0x8f, 0x0e, 0x45, 0xe1, 0x39, + 0xc7, 0x49, 0x3a, 0xfa, 0xad, 0xfa, 0x9c, 0xe6, 0x8a, 0xa8, 0x79, 0x2a, 0xcd, 0x3b, 0x3e, 0x95, + 0xcd, 0xda, 0xa7, 0x32, 0x80, 0x7b, 0xbf, 0x9c, 0x8f, 0x23, 0x29, 0x4e, 0xae, 0xe2, 0x85, 0x14, + 0xb3, 0x91, 0xc8, 0x27, 0xb8, 0x2a, 0x3b, 0xf8, 0x9b, 0xa3, 0x51, 0xb5, 0x46, 0xc8, 0xaf, 0x8c, + 0xad, 0x79, 0x4a, 0xae, 0x7e, 0x4a, 0xbe, 0x9a, 0x83, 0xcd, 0xb8, 0xaf, 0x49, 0x9c, 0xbd, 0x71, + 0x49, 0x7f, 0x5d, 0x3c, 0x8a, 0x67, 0x41, 0x7f, 0x45, 0xfd, 0x5a, 0x87, 0x65, 0xb3, 0x0e, 0x96, + 0xe0, 0x57, 0xa5, 0x0e, 0x87, 0x46, 0x8f, 0x26, 0x93, 0x4c, 0x4c, 0x22, 0xa9, 0x33, 0xc2, 0x30, + 0xd8, 0x07, 0xb0, 0x49, 0xca, 0x3a, 0xa8, 0xf5, 0x23, 0x4e, 0xae, 0x73, 0xbc, 0xf3, 0x8f, 0x37, + 0xbb, 0xce, 0x3f, 0xdf, 0xec, 0x3a, 0xff, 0x79, 0xb3, 0xeb, 0xfc, 0xe9, 0xbf, 0xbb, 0x1b, 0x97, + 0x9b, 0xf4, 0xb7, 0xef, 0xfb, 0xff, 0x0f, 0x00, 0x00, 0xff, 0xff, 0x8c, 0xa8, 0x3e, 0x99, 0xfd, + 0x13, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -5152,6 +5162,16 @@ func (m *ImportRoaringRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.UpdateExistence { + i-- + if m.UpdateExistence { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x38 + } if m.FieldCreatedAt != 0 { i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) i-- @@ -6400,6 +6420,9 @@ func (m *ImportRoaringRequest) Size() (n int) { if m.FieldCreatedAt != 0 { n += 1 + sovPublic(uint64(m.FieldCreatedAt)) } + if m.UpdateExistence { + n += 2 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -12610,6 +12633,26 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { break } } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdateExistence", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.UpdateExistence = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index 51de362bd..077ed87f9 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -251,6 +251,7 @@ message ImportRoaringRequest { uint64 Block = 4; int64 IndexCreatedAt = 5; int64 FieldCreatedAt = 6; + bool UpdateExistence = 7; } message ImportColumnAttrsRequest { diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 4abe89072..4773ef666 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -15,6 +15,9 @@ package pilosa import ( + "bytes" + "github.com/pilosa/pilosa/v2/roaring" + "reflect" "testing" ) @@ -47,10 +50,12 @@ type memAttrStore struct { store map[uint64]map[string]interface{} } -func (s *memAttrStore) Path() string { return "" } -func (s *memAttrStore) Open() error { return nil } -func (s *memAttrStore) Close() error { return nil } -func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return s.store[id], nil } +func (s *memAttrStore) Path() string { return "" } +func (s *memAttrStore) Open() error { return nil } +func (s *memAttrStore) Close() error { return nil } +func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { + return s.store[id], nil +} func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { s.store[id] = m return nil @@ -61,5 +66,33 @@ func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { } return nil } -func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil } -func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil } +func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil } +func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { + return nil, nil +} + +func TestAPI_CombineForExistence(t *testing.T) { + bm := roaring.NewBitmap() + bm.Add(pos(1, 1)) + bm.Add(pos(1, 2)) + bm.Add(pos(1, 65537)) //make sure to cross container boundary + bm.Add(pos(1, 65538)) + bm.Add(pos(2, 1)) + bm.Add(pos(2, 2)) + bm.Add(pos(2, 65537)) + bm.Add(pos(2, 65538)) + + buf := new(bytes.Buffer) + _, err := bm.WriteTo(buf) + panicOn(err) + raw := buf.Bytes() + results := combineForExistence(raw) + bm2 := roaring.NewBitmap() + bm2.ImportRoaringBits(results, false, false, 1< Date: Fri, 19 Feb 2021 18:08:25 -0600 Subject: [PATCH 03/28] Provide option to update existence on import roaring --- api.go | 23 +- cmd/sauron/design.org | 199 ---- cmd/sauron/httputil.go | 274 ----- cmd/sauron/main.go | 1802 --------------------------------- cmd/sauron/populate.sh | 10 - cmd/sauron/sample_schema.json | 1 - cmd/sauron/sauron_test.go | 396 -------- cmd/sauron/schema_test.go | 353 ------- cmd/sauron/sqlgen_test.go | 260 ----- cmd/sauron/test_sqlite.sh | 51 - cmd/sauron/vprint.go | 167 --- go.mod | 25 +- go.sum | 85 +- http/client_test.go | 6 + pilosa_internal_test.go | 35 +- 15 files changed, 109 insertions(+), 3578 deletions(-) delete mode 100644 cmd/sauron/design.org delete mode 100644 cmd/sauron/httputil.go delete mode 100644 cmd/sauron/main.go delete mode 100644 cmd/sauron/populate.sh delete mode 100644 cmd/sauron/sample_schema.json delete mode 100644 cmd/sauron/sauron_test.go delete mode 100644 cmd/sauron/schema_test.go delete mode 100644 cmd/sauron/sqlgen_test.go delete mode 100755 cmd/sauron/test_sqlite.sh delete mode 100644 cmd/sauron/vprint.go diff --git a/api.go b/api.go index 224c6bf15..f67ecf100 100644 --- a/api.go +++ b/api.go @@ -431,8 +431,15 @@ func importWorker(importWork chan importJob) { } if j.req.UpdateExistence { if ef := j.field.idx.existenceField(); ef != nil { - existence := combineForExistence(data) - ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false) + existence, err := combineForExistence(data) + if err != nil { + return errors.Wrap(err, "merging existence on roaring import") + } + + err = ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false) + if err != nil { + return errors.Wrap(err, "updating existence on roaring import") + } } } @@ -458,19 +465,21 @@ func importWorker(importWork chan importJob) { } // merge all rows to singled existence row -func combineForExistence(inputRoaringData []byte) []byte { +func combineForExistence(inputRoaringData []byte) ([]byte, error) { rowSize := uint64(1 << shardVsContainerExponent) rit, err := roaring.NewRoaringIterator(inputRoaringData) if err != nil { - panicOn(err) + return nil, err } bm := roaring.NewBitmap() - bm.MergeRoaringRawIteratorIntoExists(rit, rowSize) + err = bm.MergeRoaringRawIteratorIntoExists(rit, rowSize) + if err != nil { + return nil, err + } buf := new(bytes.Buffer) _, err = bm.WriteTo(buf) - panicOn(err) - return buf.Bytes() + return buf.Bytes(), err } // ImportRoaring is a low level interface for importing data to Pilosa when diff --git a/cmd/sauron/design.org b/cmd/sauron/design.org deleted file mode 100644 index aa579ab34..000000000 --- a/cmd/sauron/design.org +++ /dev/null @@ -1,199 +0,0 @@ - -* Oracle Design -- setup server that intersepts PQL queries and proxies to live pilosa server -- implement all the pilosa functions using SQL, specifically the sqlite flavor. This will - provide validation against pilosa results -- provide timings for each query path, PQL and SQL and compare -- this is intended to be a correctness check and not a performance check. the - sql solution is likely to be pretty slow with realitively small numbers ~1millions - #+BEGIN_SRC - .─────. - ,' `. ┌────────────┐ ┌────────┐ -;pilosa json: │ Sauron │ ┌──▶│ pilosa │ -: client ;──────────▶ │ │───┤ └────────┘ - ╲ ╱ │ │ │ ┌────────┐ - `. ,' └────────────┘ └──▶│ sqlite │ - `───' └────────┘ - #+END_SRC -* Setup Sql (sqlite) -#+name: setup test -#+header: :results silent -#+header: :db poc.db - #+BEGIN_SRC sqlite - create table bits (field string,row string, column bigint); - create table bsi (field string,column bigint , val bigint); - CREATE VIEW columns as - select distinct column from bits - UNION - select distinct column from bsi; - - INSERT INTO bits VALUES( 'color','red',1); - INSERT INTO bits VALUES( 'color','red',2); - INSERT INTO bits VALUES( 'color','red',3); - INSERT INTO bits VALUES( 'color','red',4); - INSERT INTO bits VALUES( 'color','red',5); - INSERT INTO bits VALUES( 'color','green',2); - INSERT INTO bits VALUES( 'color','green',4); - INSERT INTO bits VALUES( 'color','green',6); - INSERT INTO bits VALUES( 'color','green',8); - INSERT INTO bits VALUES( 'color','yellow',1); - INSERT INTO bits VALUES( 'color','yellow',3); - INSERT INTO bits VALUES( 'color','yellow',5); - INSERT INTO bits VALUES( 'color','yellow',7); - INSERT INTO bits VALUES( 'color','orange',5); - INSERT INTO bits VALUES( 'color','orange',6); - INSERT INTO bits VALUES( 'color','orange',7); - INSERT INTO bits VALUES( 'color','orange',8); - - INSERT INTO bsi VALUES( 'luminosity',1, 1); - INSERT INTO bsi VALUES( 'luminosity',2, 2); - INSERT INTO bsi VALUES( 'luminosity',3, 4); - INSERT INTO bsi VALUES( 'luminosity',4, 5); - INSERT INTO bsi VALUES( 'luminosity',5, 4); - INSERT INTO bsi VALUES( 'luminosity',6, 3); - INSERT INTO bsi VALUES( 'luminosity',7, 2); - INSERT INTO bsi VALUES( 'luminosity',8, 1); - #+END_SRC - -* TODO PQL to SQL TODO [10/11] -- [X] Row(color=red) - #+begin_src go -(*pql.Query)(&pql.Query{ - Calls: ([]*pql.Call)([]*pql.Call{ - (*pql.Call)(&pql.Call{ - Name: (string)("Row"), - Args: (map[string]interface{})(map[string]interface{}{ - (string)("color"): (string)("red"), - }), - Children: ([]*pql.Call)(nil), - Type: (pql.CallType)(0), - Precomputed: (map[uint64]interface{})(nil), - }), - }), - callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), - conditional: ([]string)(nil), -}) - #+end_src - #+BEGIN_SRC sqlite - select column from bits where field="color" AND row="red" - #+END_SRC -- [X] Count(Row(color=red)) - #+begin_src go -(*pql.Query)(&pql.Query{ - Calls: ([]*pql.Call)([]*pql.Call{ - (*pql.Call)(&pql.Call{ - Name: (string)("Count"), - Args: (map[string]interface{})(nil), - Children: ([]*pql.Call)([]*pql.Call{ - (*pql.Call)(&pql.Call{ - Name: (string)("Row"), - Args: (map[string]interface{})(map[string]interface{}{ - (string)("color"): (string)("red"), - }), - Children: ([]*pql.Call)(nil), - Type: (pql.CallType)(0), - Precomputed: (map[uint64]interface{})(nil), - }), - }), - Type: (pql.CallType)(0), - Precomputed: (map[uint64]interface{})(nil), - }), - }), - callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), - conditional: ([]string)(nil), -}) - - #+end_src - #+BEGIN_SRC sql - select count(*) from( - select column from bits where field="color" AND row="red" - ) - #+END_SRC -- [X] Intersect(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from bits where field="color" AND row="red" - intersect - select column from bits where field="color" AND row="green" - #+END_SRC -- [X] Union(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from bits where field="color" AND row="red" - union - select column from bits where field="color" AND row="green" - #+END_SRC -- [X] Difference(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from bits where field="color" AND row="red" - except - select column from bits where field="color" AND row="green" - #+END_SRC -- [X] Not(Row(color=red)) - #+BEGIN_SRC sql - select column from columns - except - select column from bits where field="color" AND row="red" - #+END_SRC -- [X] Xor(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from ( - select column from bits where field="color" AND row="green" - union - select column from bits where field="color" AND row="red" - ) - except - select column from ( - select column from bits where field="color" AND row="green" - intersect - select column from bits where field="color" AND row="red" - ) - #+END_SRC -- [X] Distinct(field=luminosity) - #+BEGIN_SRC sql - select distinct val from bsi where field="lumin" - #+END_SRC -- [X] Distinct(Row(color="red"),field=luminosity) - #+BEGIN_SRC sql - select distinct val from bsi - where field="luminosity" ANDcolumn in ( - select column from bits where row="color" AND row="red" - ) - #+END_SRC -- [X] Distinct(Intersect(Row(color="red"),Row(color="green")),luminosity) - #+BEGIN_SRC sql - select distinct val from bsi - where column in ( - select column from bits where field="color" AND row="red" - intersect - select column from bits where field="color" AND row="green" - ) - AND field="luminosity" - #+END_SRC - -- [X] Count(Union(Intersect(Row(color=red),Row(color=green)),Intersect(Row(color=yellow),Row(color=orange)))) -#+begin_src sqlite -select count(*) from ( -select column from ( - select column from bits where field="color" AND row="green" - intersect - select column from bits where field="color" AND row="red" -) -union -select column from ( - select column from bits where field="color" AND row="yellow" - intersect - select column from bits where field="color" AND row="orange" -)) - -- [ ] Rows(color) -#+begin_src sqlite -select distinct row from bits where field="color" -#+end_src - -- [ ] GroupBy(Rows(color)) - #+BEGIN_SRC sql - select row,count(*) from bits where field="color" group by row; - #+END_SRC -- [ ] GroupBy(Rows(color), Rows(other), limit=7) - #+BEGIN_SRC sql - select row,count(*) from bits where field="color" group by row; - #+END_SRC diff --git a/cmd/sauron/httputil.go b/cmd/sauron/httputil.go deleted file mode 100644 index 7056d40e8..000000000 --- a/cmd/sauron/httputil.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/json" - "log" - "mime" - "net/http" - "strings" - - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/pql" - "github.com/pkg/errors" -) - -// validHeaderAcceptJSON returns false if one or more Accept -// headers are present, but none of them are "application/json" -// (or any matching wildcard). Otherwise returns true. -func validHeaderAcceptJSON(header http.Header) bool { - return validHeaderAcceptType(header, "application", "json") -} - -func validHeaderAcceptType(header http.Header, typ, subtyp string) bool { - if v, found := header["Accept"]; found { - for _, v := range v { - t, _, err := mime.ParseMediaType(v) - if err != nil { - switch err { - case mime.ErrInvalidMediaParameter: - // This is an optional feature, so we can keep going anyway. - default: - continue - } - } - spl := strings.SplitN(t, "/", 2) - if len(spl) < 2 { - continue - } - switch { - case spl[0] == typ && spl[1] == subtyp: - return true - case spl[0] == "*" && spl[1] == subtyp: - return true - case spl[0] == typ && spl[1] == "*": - return true - case spl[0] == "*" && spl[1] == "*": - return true - } - } - return false - } - return true -} - -// successResponse is a general success/error struct for http responses. -type successResponse struct { - //h *Handler - Success bool `json:"success"` - Name string `json:"name,omitempty"` - CreatedAt int64 `json:"createdAt,omitempty"` - //Error *Error `json:"error,omitempty"` - Error error -} - -// check determines success or failure based on the error. -// It also returns the corresponding http status code. -func (r *successResponse) check(err error) (statusCode int) { - if err == nil { - r.Success = true - return 0 - } - - cause := errors.Cause(err) - - // Determine HTTP status code based on the error type. - switch cause.(type) { - case pilosa.BadRequestError: - statusCode = http.StatusBadRequest - case pilosa.ConflictError: - statusCode = http.StatusConflict - case pilosa.NotFoundError: - statusCode = http.StatusNotFound - default: - statusCode = http.StatusInternalServerError - } - - r.Success = false - r.Error = err // = &Error{Message: err.Error()} - - return statusCode -} - -// write sends a response to the http.ResponseWriter based on the success -// status and the error. -func (r *successResponse) write(w http.ResponseWriter, err error) { - // Apply the error and get the status code. - statusCode := r.check(err) - - // Marshal the json response. - msg, err := json.Marshal(r) - if err != nil { - http.Error(w, string(msg), http.StatusInternalServerError) - return - } - - // Write the response. - if statusCode == 0 { - w.Header().Set("Content-Type", "application/json") - _, err := w.Write(msg) - if err != nil { - log.Printf("error writing response: %v", err) - return - } - _, err = w.Write([]byte("\n")) - if err != nil { - log.Printf("error writing newline after response: %v", err) - return - } - } else { - http.Error(w, string(msg), statusCode) - } -} - -type postFieldRequest struct { - Options fieldOptions `json:"options"` -} - -// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values, -// and used for input validation. -type fieldOptions struct { - Type string `json:"type,omitempty"` - CacheType *string `json:"cacheType,omitempty"` - CacheSize *uint32 `json:"cacheSize,omitempty"` - Min *pql.Decimal `json:"min,omitempty"` - Max *pql.Decimal `json:"max,omitempty"` - Scale *int64 `json:"scale,omitempty"` - TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` - Keys *bool `json:"keys,omitempty"` - NoStandardView bool `json:"noStandardView,omitempty"` - ForeignIndex *string `json:"foreignIndex,omitempty"` -} - -func (o *fieldOptions) ToPilosaFieldOptions() *pilosa.FieldOptions { - - fo := &pilosa.FieldOptions{ - Type: o.Type, - - /* - //Base int64 `json:"base,omitempty"` - //BitDepth uint `json:"bitDepth,omitempty"` - Min: *o.Min, - Max: *o.Max, - Scale: *o.Scale, - NoStandardView: o.NoStandardView, - CacheSize: *o.CacheSize, - CacheType: *o.CacheType, - TimeQuantum: *o.TimeQuantum, - ForeignIndex: *o.ForeignIndex, - */ - } - if o.Keys != nil { - fo.Keys = *o.Keys - } - return fo -} - -func (o *fieldOptions) validate() error { - // Pointers to default values. - defaultCacheType := pilosa.DefaultCacheType - defaultCacheSize := uint32(pilosa.DefaultCacheSize) - - switch o.Type { - case pilosa.FieldTypeSet, "": - // Because FieldTypeSet is the default, its arguments are - // not required. Instead, the defaults are applied whenever - // a value does not exist. - if o.Type == "" { - o.Type = pilosa.FieldTypeSet - } - if o.CacheType == nil { - o.CacheType = &defaultCacheType - } - if o.CacheSize == nil { - o.CacheSize = &defaultCacheSize - } - if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type set")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type set")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) - } - case pilosa.FieldTypeInt: - if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) - } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { - return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) - } - case pilosa.FieldTypeDecimal: - if o.Scale == nil { - return pilosa.NewBadRequestError(errors.New("decimal field requires a scale argument")) - } else if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) - } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { - return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) - } - case pilosa.FieldTypeTime: - if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time")) - } else if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type time")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type time")) - } else if o.TimeQuantum == nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time")) - } - case pilosa.FieldTypeMutex: - if o.CacheType == nil { - o.CacheType = &defaultCacheType - } - if o.CacheSize == nil { - o.CacheSize = &defaultCacheSize - } - if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type mutex")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) - } - case pilosa.FieldTypeBool: - if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) - } else if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) - } else if o.Keys != nil { - return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool")) - } else if o.ForeignIndex != nil { - return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key")) - } - default: - return errors.Errorf("invalid field type: %s", o.Type) - } - return nil -} diff --git a/cmd/sauron/main.go b/cmd/sauron/main.go deleted file mode 100644 index 497e5346b..000000000 --- a/cmd/sauron/main.go +++ /dev/null @@ -1,1802 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - //"io" - "bytes" - "context" - "database/sql" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "log" - "net" - "net/http" - "os" - "os/exec" - "sort" - "strconv" - "strings" - "time" - - "modernc.org/sqlite" - _ "modernc.org/sqlite" - - //"github.com/gorilla/handlers" - "github.com/gorilla/mux" - "github.com/pilosa/pilosa/v2" - - "github.com/pilosa/pilosa/v2/encoding/proto" - "github.com/pilosa/pilosa/v2/pql" - "github.com/pilosa/pilosa/v2/roaring" - "github.com/pkg/errors" - "github.com/shurcooL/go-goon" -) - -func main() { - -} - -// Sauron sees everything. -// And he converts PQL into SQL. -// -// The PQL is applied to a Pilosa and the SQL -// is applied to a SQL database. Then the -// query results are compared as they are -// returned to the client. -// -type Sauron struct { - cfg *SauronConfig - - sql *PQLToSQL - url string // "https://..." - addr string // host:port - mux *http.ServeMux - srv *http.Server - cmd *exec.Cmd - - // target of proxy - pilosaHttpURL string - - Handler http.Handler -} - -type SauronConfig struct { - Bind string // host:port - BindGRPC string // host:port - GossipPort string // port - DataDir string - - DatabaseName string // for sql database -} - -// IndexInfo2 and FieldInfo2 let us look up -// the schema details quickly by using maps -// to index the fields in an index. -type IndexInfo2 struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options pilosa.IndexOptions `json:"options"` - Fields map[string]*FieldInfo2 `json:"fields"` - ShardWidth uint64 `json:"shardWidth"` -} - -type FieldInfo2 struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options pilosa.FieldOptions `json:"options"` - Views []*pilosa.ViewInfo `json:"views,omitempty"` - indexInfo *IndexInfo2 -} - -func (fi *FieldInfo2) GetIndexName() string { - return fi.indexInfo.Name -} - -func (fi *FieldInfo2) GetIndexCreatedAt() int64 { - return fi.indexInfo.CreatedAt -} - -func (fi *FieldInfo2) GetCreatedAt() int64 { - return fi.CreatedAt -} - -func (fi *FieldInfo2) GetName() string { - return fi.Name -} - -type PQLToSQL struct { - schema pilosa.Schema - DB *sql.DB - - dbprefix string // prefixed to all tables - - // map from indexName to *IndexInfo2 - i2f map[string]*IndexInfo2 - ShardWidth uint64 -} - -func NewPQLToSQL(dbprefix string) *PQLToSQL { - return &PQLToSQL{ - dbprefix: strings.ToLower(dbprefix), // some SQL are not case sensitive, just use lower case. - i2f: make(map[string]*IndexInfo2), - ShardWidth: 1048576, - } -} - -func (ps *PQLToSQL) Start() error { - db, err := sql.Open("sqlite", fmt.Sprintf("%v.db", ps.dbprefix)) - if err != nil { - log.Fatal(err) - } - ps.DB = db - return nil -} -func (ps *PQLToSQL) MustRemove() { - name := fmt.Sprintf("%v.db", ps.dbprefix) - os.Remove(name) - // panicOn(err) -} - -func (ps *PQLToSQL) Stop() error { - return ps.DB.Close() -} -func (ps *PQLToSQL) Store(index, field string, rowIdOrKey interface{}, rowCall []*pql.Call) (bool, error) { - rowSql, _, _, err := ToSql(rowCall, index) - panicOn(err) - insertSql := fmt.Sprintf(`insert into %vλbits select "%v" as field, "%v" as row, column,0 as timestamp from ( %v )`, index, field, rowIdOrKey, rowSql) - res, err := ps.DB.Exec(insertSql) - if err != nil { - return false, err - } - aff, err := res.RowsAffected() - if err != nil { - return false, err - } - return aff > 0, nil -} - -// convert '-' to 'Θ' since db cannot have '-' in table names, -// but pilosa allows it in index and field names. -func dash2theta(s string) (r string) { - return strings.Replace(s, "-", "Θ", -1) -} - -func theta2dash(s string) (r string) { - return strings.Replace(s, "Θ", "-", -1) -} -func (ps *PQLToSQL) CreateSchema(jsonBytes []byte) (err error) { - err = json.Unmarshal(jsonBytes, &ps.schema) - if err != nil { - panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) - } - for _, i := range ps.schema.Indexes { - ps.createDatabase(i.Name) - for _, f := range i.Fields { - panicOn(ps.CreateField(i, f)) - } - } - return nil -} - -func (ps *PQLToSQL) CreateFieldWithOptions(indexName, fieldName string, o fieldOptions) error { - - ii2, ok := ps.i2f[indexName] - if !ok { - return fmt.Errorf("index '%v' not found", indexName) - } - - f2, ok := ii2.Fields[fieldName] - if !ok { - f2 = &FieldInfo2{ - Name: fieldName, - //CreatedAt: f.CreatedAt, - Options: *(o.ToPilosaFieldOptions()), - //Views: f.Views, - indexInfo: ii2, - } - ii2.Fields[fieldName] = f2 - } - return nil -} - -func (ps *PQLToSQL) CreateIndex(indexName string, opts *pilosa.IndexOptions) (err error) { - ii2, ok := ps.i2f[indexName] - if !ok { - ii2 = &IndexInfo2{ - Name: indexName, - //CreatedAt: i.CreatedAt, - Options: *opts, - Fields: make(map[string]*FieldInfo2), - ShardWidth: 1048576, - } - //TODO (twg) figure oiut where to get Shardwidht - ps.i2f[indexName] = ii2 - } - return nil -} -func (ps *PQLToSQL) Query(index string, query []*pql.Call) (err error) { - // sql, err := ToSQL(query, index) - //TODO (twg) implement QUery - return nil -} -func (ps *PQLToSQL) CreateField(i *pilosa.IndexInfo, f *pilosa.FieldInfo) (err error) { - ii2, ok := ps.i2f[i.Name] - if !ok { - ii2 = &IndexInfo2{ - Name: i.Name, - CreatedAt: i.CreatedAt, - Options: i.Options, - Fields: make(map[string]*FieldInfo2), - ShardWidth: i.ShardWidth, - } - ps.i2f[i.Name] = ii2 - } - - f2, ok := ii2.Fields[f.Name] - if !ok { - f2 = &FieldInfo2{ - Name: f.Name, - CreatedAt: f.CreatedAt, - Options: f.Options, - Views: f.Views, - indexInfo: ii2, - } - ii2.Fields[f.Name] = f2 - } - return nil -} - -func (ps *PQLToSQL) createDatabase(index string) (err error) { - idx_prefix := dash2theta(index) + "λ" - // Create fields that don't exist. - sql := fmt.Sprintf("create table %vbits (field string,row,column,timestamp, unique(field,row,column,timestamp) )", idx_prefix) - _, err = ps.DB.Exec(sql) - panicOn(err) - sql = fmt.Sprintf("create table %vbsi (field,column, val bigint,unique(field,column)) ", idx_prefix) - _, err = ps.DB.Exec(sql) - panicOn(err) - sql = fmt.Sprintf(`CREATE VIEW %vcolumns as - select distinct column from %vbits - UNION - select distinct column from %vbsi`, idx_prefix, idx_prefix, idx_prefix) - _, err = ps.DB.Exec(sql) - panicOn(err) - return -} -func (ps *PQLToSQL) GetField(indexName, fieldName string) (*FieldInfo2, error) { - idx, ok := ps.i2f[indexName] - if !ok { - return nil, errors.New(fmt.Sprintf("index not found '%v'", indexName)) - } - f, ok := idx.Fields[fieldName] - if !ok { - return nil, errors.New(fmt.Sprintf("field not found '%v' in index '%v'", fieldName, indexName)) - } - return f, nil -} - -//TODO (twg) below will be moved to its own file later just have it here for quick navigation -type PqlFunction uint8 - -//debate between this -type ResultType uint8 - -const ( - UNKNOWN = ResultType(iota) - BITMAP - INT - ARRAY - GROUPCOUNT - SIGNEDROW - BOOL - PAIRSFIELD - TABLE - AGGSQL -) - -func getAggregateSql(call *pql.Call, index string) (string, error) { - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", errors.New("no _field present") - } - } - var sq string - if len(call.Children) > 0 { - //only child should be a bitmap call - var err error - sq, _, _, err = ToSql(call.Children, index) - if err != nil { - return sq, err - } - sq = " and column in (" + sq + ")" - - } - sqb := fmt.Sprintf(`select %v(val) from %vλbsi where field="%v"%v`, strings.ToLower(call.Name), index, fieldName, sq) - sql := fmt.Sprintf(`select val,count(*) from %vλbsi where val=(%v)%v`, index, sqb, sq) - - return sql, nil -} - -// ToSql takes a pilosa query(ast) and converts it to sql -// assumption: the pql has successfully executed on server -func applyCondition(field string, cond *pql.Condition) (sql string) { - - switch cond.Op { - case pql.EQ, pql.NEQ, pql.LT, pql.LTE, pql.GT, pql.GTE: - val, ok := cond.Uint64Value() - if !ok { - panic("no value for condition") - } - if cond.Op == pql.EQ { - sql = fmt.Sprintf(`field="%v" and val=%d`, field, val) - } else if cond.Op == pql.NEQ { - sql = fmt.Sprintf(`field="%v" and val!=%d`, field, val) - } else if cond.Op == pql.LT { - sql = fmt.Sprintf(`field="%v" and val<%d`, field, val) - } else if cond.Op == pql.LTE { - sql = fmt.Sprintf(`field="%v" and val<=%d`, field, val) - } else if cond.Op == pql.GT { - sql = fmt.Sprintf(`field="%v" and val>%d`, field, val) - } else if cond.Op == pql.GTE { - sql = fmt.Sprintf(`field="%v" and val>=%d`, field, val) - } - case pql.BETWEEN, pql.BTWN_LT_LTE, pql.BTWN_LTE_LT, pql.BTWN_LT_LT: - val, ok := cond.Uint64SliceValue() - if !ok { - panic("bad value for condition") - } - if cond.Op == pql.BETWEEN { - sql = fmt.Sprintf(`field="%v" and %v>=%d and %v<=%d`, field, "val", val[0], "val", val[1]) - } else if cond.Op == pql.BTWN_LT_LTE { - sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[1]) - } else if cond.Op == pql.BTWN_LTE_LT { - sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[0]) - } else if cond.Op == pql.BTWN_LT_LT { - sql = fmt.Sprintf(`field="%v" and %v>%d and %v<%d`, field, "val", val[0], "val", val[1]) - } - } - return -} -func FetchFields(rowsCalls []*pql.Call) ([]string, []interface{}, error) { - //TODO (twg) date fields in rows? - results := make([]string, len(rowsCalls)) - columns := make([]interface{}, 0) - for i, call := range rowsCalls { - //TODO (twg) Time Fields ie from/to args - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return nil, nil, errors.New("no field for rows") - } - } - results[i] = fieldName.(string) - col, ok := call.Args["column"] - if ok { - s := fmt.Sprintf("f%v.column='%v'", i+1, col) - columns = append(columns, s) - } - } - return results, columns, nil -} - -func ToSql(ast []*pql.Call, index string) (string, ResultType, []interface{}, error) { - resultType := UNKNOWN - optional := make([]interface{}, 0) - if len(ast) == 0 { - return "", resultType, optional, nil - } - stack := make([]string, 0) - for _, call := range ast { - switch c := strings.ToLower(call.Name); c { - case "row": - var sql string - if call.HasConditionArg() { - for k, v := range call.Args { - csql := applyCondition(k, v.(*pql.Condition)) - sql = fmt.Sprintf("select distinct column from %vλbsi where %v", index, csql) - - break - } - } else { - field, err := call.FieldArg() - panicOn(err) - row := call.Args[field] - - t, ok := call.Args["from"] - var timestampClause string - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause = fmt.Sprintf(` AND timestamp >= "%v"`, clean) - } - t, ok = call.Args["to"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause += fmt.Sprintf(` AND timestamp < "%v"`, clean) - - } - - sql = fmt.Sprintf(`select distinct column from %vλbits where field="%v" AND row="%v"%v`, index, field, row, timestampClause) - - } - stack = append(stack, sql) - resultType = BITMAP - case "count": - //should have args and should be top level - if len(call.Children) == 0 { - return "", resultType, optional, errors.New("invalid args for count") - } - nestedSQL, _, _, err := ToSql(call.Children, index) - panicOn(err) - sql := "select count(*) from( " + nestedSQL + " )" - stack = append(stack, sql) - resultType = INT - case "union", "intersect": - var sql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(sql) == 0 { - sql = sq - } else { - sql += " " + c + " " + sq - } - } - sql = "select column from(" + sql + ")" - stack = append(stack, sql) - resultType = BITMAP - case "difference": - var sql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(sql) == 0 { - sql = sq - } else { - sql += " except " + sq - } - } - sql = "select column from(" + sql + ")" - stack = append(stack, sql) - resultType = BITMAP - case "not": - //should have args and should be top level - if len(call.Children) == 0 { - return "", resultType, optional, errors.New("invalid args for count") - } - nestedSQL, _, _, err := ToSql(call.Children, index) - panicOn(err) - sql := fmt.Sprintf("select column from %vλcolumns except %v", index, nestedSQL) - stack = append(stack, sql) - resultType = BITMAP - case "xor": - var unionSql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(unionSql) == 0 { - unionSql = sq - } else { - unionSql += " union " + sq - } - } - unionSql = "select column from(" + unionSql + ")" - var intersectSql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(intersectSql) == 0 { - intersectSql = sq - } else { - intersectSql += " intersect " + sq - } - } - intersectSql = "select column from(" + intersectSql + ")" - sql := unionSql + " except " + intersectSql - stack = append(stack, sql) - resultType = BITMAP - - case "distinct": - //TODO validate field type, only valid for bsi fields - fieldName, ok := call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("field not specified on distinct call") - } - var sql string - if len(call.Children) > 1 { - return "", resultType, optional, errors.New(fmt.Sprintf("unexpected args to distinct query:'%v'", call.Children)) - } - sql = fmt.Sprintf(`select distinct val from %vλbsi where field="%v"`, index, fieldName) - if len(call.Children) > 0 { - sq, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) - panicOn(err) - sql += " AND column in (" + sq + ")" - } - stack = append(stack, sql) - resultType = SIGNEDROW - case "rows": - //TODO (twg) Time Fields ie from/to args - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("no _field present") - } - } - optional = append(optional, fieldName) - sql := fmt.Sprintf(`select distinct field, row from %vλbits where field="%v"`, index, fieldName) - col, ok := call.Args["column"] - if ok { - sql = fmt.Sprintf("%v and column='%v'", sql, col) - } - prev, ok := call.Args["previous"] - if ok { - sql = fmt.Sprintf("%v and row>'%v'", sql, prev) - } - t, ok := call.Args["from"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) - sql += timestampClause - } - t, ok = call.Args["to"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) - sql += timestampClause - } - sql += " order by row" - limit, ok := call.Args["limit"] - if ok { - sql = fmt.Sprintf("%v limit %v", sql, limit) - } - stack = append(stack, sql) - resultType = ARRAY - case "groupby": - //need to figure out how many fields we will be looking at - fields, optColumns, err := FetchFields(call.Children) - panicOn(err) - sql := "select " - for i := range fields { - f := fmt.Sprintf("f%d", i+1) - if i == 0 { - sql += fmt.Sprintf(" %s.field, %s.row", f, f) - } else { - - sql += fmt.Sprintf(",%s.field, %s.row", f, f) - } - } - tname := fmt.Sprintf("%vλbits", index) - sql += fmt.Sprintf(",count(*) as cnt from %v f1", tname) - if len(fields) > 1 { - for i := range fields[1:] { - f := fmt.Sprintf("f%d", i+2) - sql += fmt.Sprintf(" inner join %v %v on f1.column = %v.column", tname, f, f) - } - } - sql += " where " - gb := "" - for i, field := range fields { - f := fmt.Sprintf("f%d", i+1) - if i == 0 { - sql += fmt.Sprintf("%v.field='%v'", f, field) - gb += fmt.Sprintf("%v.field,%v.row", f, f) - } else { - sql += fmt.Sprintf("and %v.field='%v'", f, field) - gb += fmt.Sprintf(",%v.field,%v.row", f, f) - } - } - if len(optColumns) > 0 { - for _, column := range optColumns { - sql += fmt.Sprintf(" and %v ", column) - } - } - sql += " group by " + gb + " order by cnt desc" - - optional = append(optional, len(fields)) - - stack = append(stack, sql) - resultType = GROUPCOUNT - case "topn": - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("no _field present") - } - } - var sq string - if len(call.Children) > 0 { - if len(call.Children) > 1 { - return "", resultType, optional, errors.New("invalid topn") - } - var err error - sq, _, _, err = ToSql(call.Children, index) - panicOn(err) - sq = " and column in (" + sq + ")" - - } - sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) - - limit, ok := call.Args["n"] - if ok { - sql += fmt.Sprintf(" limit %v", limit) - } - stack = append(stack, sql) - resultType = PAIRSFIELD - case "topk": - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("no _field present") - } - } - var sq string - filter, ok := call.Args["filter"] - if ok { - //assuming this works in pilosa so the filter must be a correct bitmap call - //ie row/union/intersect/xor/difference - var err error - sq, _, _, err = ToSql([]*pql.Call{filter.(*pql.Call)}, index) - panicOn(err) - sq = " and column in (" + sq + ")" - - } - t, ok := call.Args["from"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) - sq += timestampClause - } - t, ok = call.Args["to"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) - sq += timestampClause - } - sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) - - limit, ok := call.Args["k"] - if ok { - sql += fmt.Sprintf(" limit %v", limit) - } - stack = append(stack, sql) - resultType = PAIRSFIELD - case "min", "max": - sql, err := getAggregateSql(call, index) - panicOn(err) - stack = append(stack, sql) - resultType = AGGSQL - case "sum": - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - panic("no _field present") - } - } - var sq string - if len(call.Children) > 0 { - //only child should be a bitmap call - var err error - sq, _, _, err = ToSql(call.Children, index) - panicOn(err) - sq = " and column in (" + sq + ")" - - } - sql := fmt.Sprintf(`select sum(val),count(*) from %vλbsi where field="%v"%v`, index, fieldName, sq) - stack = append(stack, sql) - resultType = AGGSQL - case "constrow": - sql := "select column1 as column from (values " - columns, ok := call.Args["columns"].([]interface{}) - - if ok { - for i, col := range columns { - if i != 0 { - sql += "," - } - sql += fmt.Sprintf("(%v)", col) - - } - sql += ")" - - } - - stack = append(stack, sql) - resultType = BITMAP - case "all": - sql := fmt.Sprintf("select column from %vλcolumns", index) - stack = append(stack, sql) - resultType = BITMAP - case "extract": - // extract is a table response - bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) - panicOn(err) - var rowsPart string - if len(call.Children) > 1 { - rows := call.Children[1:] - rowsPart = "select field,row from(" - for i, row := range rows { - part, _, _, err1 := ToSql([]*pql.Call{row}, index) - panicOn(err1) - if i > 0 { - rowsPart += "union all select field,row from (" - } - rowsPart += part + ")" - } - } - var sql string - if len(rowsPart) == 0 { - sql = "select '' as field, '' as row,column from (" + bm + ")" - } else { - sql = "select sq.field, sq.row, b.column from (" - sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) - sql += " where b.column in (" + bm + ")" - } - - stack = append(stack, sql) - resultType = TABLE - case "unionrows": - var rowsPart string - rowsPart = "select field,row from(" - for i, row := range call.Children { - part, _, _, err1 := ToSql([]*pql.Call{row}, index) - panicOn(err1) - if i > 0 { - rowsPart += "union all select field,row from (" - } - rowsPart += part + ")" - } - sql := "select distinct b.column from (" - sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) - stack = append(stack, sql) - resultType = BITMAP - case "includescolumn": - bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) - panicOn(err) - col, ok := call.Args["column"] - if !ok { - return "", resultType, optional, errors.New("missing column parameter") - } - - sql := fmt.Sprintf("select count(*)>0 from (select column from(%v) where column='%v')", bm, col) - stack = append(stack, sql) - resultType = BOOL - default: - goon.Dump(call) - return "", resultType, optional, errors.New(fmt.Sprintf("unknown call '%v'", c)) - } - } - if len(stack) == 0 { - //not sure if i should error on an NOP - return "", resultType, optional, nil - } - return stack[0], resultType, optional, nil -} - -func NewSauron(cfg *SauronConfig) *Sauron { - - addr := "127.0.0.1:8080" - url := "http://" + addr - s := &Sauron{ - cfg: cfg, - addr: addr, - url: url, - sql: NewPQLToSQL(cfg.DatabaseName), - srv: &http.Server{ - Addr: addr, - ReadTimeout: 50 * time.Millisecond, - WriteTimeout: 10 * time.Second, - MaxHeaderBytes: 1 << 20, - }, - } - s.Handler = newRouter(s) - s.srv.Handler = s.Handler - return s -} - -func (p *Sauron) Start() (urlstring string, err error) { - started := make(chan struct{}) - - panicOn(p.sql.Start()) - - // start a pilosa server - bind := p.cfg.Bind - dataDir := p.cfg.DataDir - bindGRPC := p.cfg.BindGRPC - gossipPort := p.cfg.GossipPort - - p.cmd = exec.Command("pilosa", "server", - "--bind", bind, - "--data-dir", dataDir, - "--bind-grpc", bindGRPC, - "--gossip.port", gossipPort) - err = p.cmd.Start() - panicOn(err) - - // wait for it to be listening - p.pilosaHttpURL = "http://" + bind - ok := false - seconds := 20 - for i := 0; i < seconds; i++ { - nc, err := net.Dial("tcp", bind) - if err == nil { - nc.Close() - ok = true - break - } - time.Sleep(time.Second) - } - if !ok { - return "", fmt.Errorf("could not contact pilosa server on %v after %v seconds. pid = %v", bind, seconds, p.cmd.Process.Pid) - } - go func() { - close(started) - - err = p.srv.ListenAndServe() - if err != nil { - if strings.Contains(err.Error(), "Server closed") { - return - } - } - panicOn(err) - }() - <-started - return p.url, nil -} - -func (p *Sauron) Stop() { - panicOn(p.sql.Stop()) - err := p.srv.Shutdown(context.Background()) - panicOn(err) - // Kill pilosa - if err = p.cmd.Process.Kill(); err != nil { - vv("failed to kill process: ", err) - } -} - -// GetAvailPort asks the OS for an unused port. -// There's a race here, where the port could be grabbed by someone else -// before the caller gets to Listen on it, but we are only using -// it to find a random port for the test hang debugging. -// Moreover, in practice such races are rare. Just ask for -// it again if the port is taken. -// Uses net.Listen("tcp", ":0") to determine a free port, then -// releases it back to the OS with Listener.Close(). -func GetAvailPort() int { - l, _ := net.Listen("tcp", ":0") - r := l.Addr() - l.Close() - return r.(*net.TCPAddr).Port -} - -// newRouter creates a new mux http router. -func newRouter(handler *Sauron) http.Handler { - router := mux.NewRouter() - - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") - router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") - router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") - - router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex") - router.HandleFunc("/index/{index}/field/", handler.handlePostField).Methods("POST").Name("PostField") - router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField") - router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring") - router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport") - - return router -} - -// handleGetSchema handles GET /schema requests. -func (s *Sauron) handleGetSchema(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - _ = body - _ = resp - _ = err - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(resp.StatusCode) - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - _, err = w.Write(respBody) - panicOn(err) - - // B) there is no SQL version of get schema. - -} - -// creates an entire schema from the dumped schema. -// requires completed structures, no defaults applied -func (s *Sauron) handlePostSchema(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - - w.WriteHeader(resp.StatusCode) - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - _, err = w.Write(respBody) - panicOn(err) - - // B) SQL - panicOn(s.sql.CreateSchema(body)) -} - -// handlePostField handles /index/{index}/field/{field} requests -func (s *Sauron) handlePostIndex(w http.ResponseWriter, r *http.Request) { - body, resp, err := s.ForwardHandler(w, r) - _ = body - _ = resp - _ = err - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - indexName, ok := mux.Vars(r)["index"] - if !ok { - http.Error(w, "index name is required", http.StatusBadRequest) - return - } - req := struct { - Options pilosa.IndexOptions - }{ - Options: pilosa.IndexOptions{ - Keys: false, - TrackExistence: true, - }} - if len(body) > 0 { //only process if options are present - err = json.Unmarshal(body, &req) - panicOn(err) - } - err = s.sql.CreateIndex(indexName, &req.Options) - panicOn(err) - w.Write(respBody) -} - -// handlePostField handles /index/{index}/field/{field} requests -func (s *Sauron) handlePostField(w http.ResponseWriter, r *http.Request) { - body, resp, err := s.ForwardHandler(w, r) - _ = body - _ = resp - _ = err - indexName, ok := mux.Vars(r)["index"] - if !ok { - http.Error(w, "index name is required", http.StatusBadRequest) - return - } - - fieldName, ok := mux.Vars(r)["field"] - if !ok { - http.Error(w, "field name is required", http.StatusBadRequest) - return - } - - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - // Decode request. - var req postFieldRequest - dec := json.NewDecoder(bytes.NewReader(body)) - dec.DisallowUnknownFields() - err = dec.Decode(&req) - if err != io.EOF { - panicOn(err) - } - err = s.sql.CreateFieldWithOptions(indexName, fieldName, req.Options) - _, err = w.Write(respBody) - panicOn(err) -} - -// handlePostQuery handles /index/{index}/query requests -func (s *Sauron) handlePostQuery(w http.ResponseWriter, req *http.Request) { - - body, resp, err := s.ForwardHandler(w, req) - pilosaJSON, err := ioutil.ReadAll(resp.Body) - panicOn(err) - var pilosaResponse pilosa.QueryResponse // DEBUG - dec := json.NewDecoder(bytes.NewReader(pilosaJSON)) - err = dec.Decode(&pilosaResponse) - panicOn(err) - qreq := &pilosa.QueryRequest{ - Index: mux.Vars(req)["index"], - Query: string(body), - } - - oracleResponse, err := s.sql.IssueQuery(qreq) - - if !pilosaResponse.Equals(oracleResponse) { - //plan is to write out a useful query response indicating a discrepancy - //TODO (twg) write diff in response somehow - vv("oracle:") - prettyJson(*oracleResponse) - vv("pilosa:") - prettyJson(pilosaResponse) - panic("barf") - } -} -func prettyJson(a pilosa.QueryResponse) { - src, _ := a.MarshalJSON() - dst := &bytes.Buffer{} - if err := json.Indent(dst, src, "", " "); err != nil { - panic(err) - } - vv("%v", string(dst.Bytes())) -} - -func (s *Sauron) FieldInfo(index, field string) (*FieldInfo2, error) { - idx, ok := s.sql.i2f[index] - if ok { - f, ok := idx.Fields[field] - if ok { - return f, nil - } - return nil, errors.New(fmt.Sprintf("field not in schema '%v'", field)) - } - return nil, errors.New(fmt.Sprintf("index not in schema '%v'", index)) -} -func (s *Sauron) handlePostImport(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - _ = body - _ = resp - _ = err - // Get index and field type to determine how to handle the - // import data. - indexName := mux.Vars(req)["index"] - fieldName := mux.Vars(req)["field"] - - _ = indexName - _ = fieldName - - // If the clear flag is true, treat the import as clear bits. - q := req.URL.Query() - doClear := q.Get("clear") == "true" - doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" - _ = doClear - _ = doIgnoreKeyCheck - - info, err := s.FieldInfo(indexName, fieldName) - panicOn(err) - // Unmarshal request based on field type. - fieldType := info.Options.Type - if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal { - iv := &pilosa.ImportValueRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, iv); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - //TODO (twg) ignore columnKeys at the moment - //do importValue - for i, column := range iv.ColumnIDs { - if len(iv.Values) > 0 { - s.sql.Set(indexName, column, fieldName, iv.Values[i], "") - } else if len(iv.FloatValues) > 0 { //TODO (twg) these may not be reachable - s.sql.Set(indexName, column, fieldName, iv.FloatValues[i], "") //probably some issues here - } else if len(iv.StringValues) > 0 { - s.sql.Set(indexName, column, fieldName, iv.StringValues[i], "") - } - } - } else { - // Field type: set, time, mutex - // Marshal into request object. - ir := &pilosa.ImportRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, ir); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - //do import - action := s.sql.Set - if ir.Clear { - action = func(index string, column uint64, field string, value interface{}, ts string) (bool, error) { - return s.sql.Clear(index, column, field, value) - } - } - changed := false - _ = changed - for i, column := range ir.ColumnIDs { - ts := "" - if len(ir.Timestamps) > 0 { - t := time.Unix(ir.Timestamps[i], 0) - ts = t.Format("2006-01-02 15:04:05") - } - if len(ir.RowIDs) > 0 { - change, err := action(indexName, column, fieldName, ir.RowIDs[i], ts) - panicOn(err) - if change { - changed = true - } - } else if len(ir.RowKeys) > 0 { //TODO (twg) these may not be reachable - change, err := action(indexName, column, fieldName, ir.RowKeys[i], ts) - panicOn(err) - if change { - changed = true - } - } - } - } -} -func (s *Sauron) handlePostImportRoaring(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - _ = body - _ = resp - _ = err - // Get index and field type to determine how to handle the - // import data. - indexName := mux.Vars(req)["index"] - fieldName := mux.Vars(req)["field"] - - irr := &pilosa.ImportRoaringRequest{} - err = proto.DefaultSerializer.Unmarshal(body, irr) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - //do i need the shard? - urlVars := mux.Vars(req) - shard, err := strconv.ParseUint(urlVars["shard"], 10, 64) - if err != nil { - panicOn(err) - } - for v, blob := range irr.Views { - _ = v //ignore the view for now - rit, err := roaring.NewRoaringIterator(blob) - panicOn(err) - s.sql.ImportRoaring(indexName, fieldName, shard, rit) - } -} - -//end handler -func (s *Sauron) ForwardHandler(w http.ResponseWriter, req *http.Request) (body []byte, resp *http.Response, err error) { - // TODO (twg) replace all the instances - // we need to buffer the body if we want to read it here and send it - // in the request. - body, err = ioutil.ReadAll(req.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // you can reassign the body if you need to parse it as multipart - req.Body = ioutil.NopCloser(bytes.NewReader(body)) - - // create a new url from the raw RequestURI sent by the client - url := fmt.Sprintf("%s%s", s.pilosaHttpURL, req.RequestURI) - - proxyReq, err := http.NewRequest(req.Method, url, bytes.NewReader(body)) - - // We may want to filter some headers, otherwise we could just use a shallow copy - // proxyReq.Header = req.Header - proxyReq.Header = make(http.Header) - for h, val := range req.Header { - proxyReq.Header[h] = val - } - - resp, err = http.DefaultClient.Do(proxyReq) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return - } - return -} - -// ImportRoaring must not be a bsi field..check higher up the stack -func (ps *PQLToSQL) ImportRoaring(index, field string, shard uint64, itr roaring.RoaringIterator) { - rb := roaring.NewBitmap() - for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { - rb.Put(itrKey, synthC) - } - for _, pos := range rb.Slice() { - row, column := pos/ps.ShardWidth, (shard*ps.ShardWidth)+(pos%ps.ShardWidth) - //ps.SetWithInt64Field(index, column, field, int64(row)) - ps.Set(index, column, field, int64(row), "") - } -} - -func (ps *PQLToSQL) isBSI(index, field string) (full string, isBSI bool) { - idx, ok := ps.i2f[index] - if !ok { - panic(fmt.Errorf("isBSI: no such index '%v'", index)) - } - - fld, ok := idx.Fields[field] - if !ok { - panic(fmt.Errorf("isBSI: no such field '%v'", field)) - } - if fld.Options.Type == "int" { - return dash2theta(index) + "λbsi", true - } - return dash2theta(index) + "λbits", false -} - -// Set is the basic bit operation and value operation as well, depending on the field type -func (ps *PQLToSQL) Set(index string, column uint64, field string, value interface{}, timestamp string) (bool, error) { - table, isBSI := ps.isBSI(index, field) - var sql string - if isBSI { - sql = fmt.Sprintf("INSERT INTO %v VALUES( '%v','%v', '%v')", table, field, column, value) - } else { - //if len(timestamp) == 0 { - // sql = fmt.Sprintf("INSERT INTO %v (field,row,column)VALUES( '%v','%v', '%v')", table, field, value, column) - // } else { - //TODO(twg) will need to make the timestamp as granular as the field is configured. - sql = fmt.Sprintf("INSERT INTO %v (field,row,column,timestamp)VALUES( '%v','%v', '%v','%v')", table, field, value, column, timestamp) - // } - } - - _, err := ps.DB.Exec(sql) - if err != nil { - if sqerr, ok := err.(*sqlite.Error); ok { - if sqerr.Code() == 2067 { //constraint error - return false, nil - } - } - } - panicOn(err) - return true, nil -} - -// Clear removes the specified column or value -func (ps *PQLToSQL) Clear(index string, column uint64, field string, value interface{}) (bool, error) { - table, isBSI := ps.isBSI(index, field) - var sql string - if isBSI { - sql = fmt.Sprintf("delete from %v where field='%v' and column='%v'", table, field, column) - } else { - sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' and column='%v'", table, field, value, column) - } - - res, err := ps.DB.Exec(sql) - panicOn(err) - r, err := res.RowsAffected() - panicOn(err) - if r == 0 { - return false, nil - } - return true, nil -} - -// ClearRow removes entire Row(feature) -func (ps *PQLToSQL) ClearRow(index string, field string, row interface{}) (bool, error) { - table, isBSI := ps.isBSI(index, field) - var sql string - if isBSI { - panic("bsi can't clear row allowed") - } else { - sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' ", table, field, row) - } - - r, err := ps.DB.Exec(sql) - panicOn(err) - n, err := r.RowsAffected() - return n > 0, err -} - -func sliceToString(slc []int64) (r string) { - if len(slc) == 0 { - return - } - for _, i := range slc { - r += fmt.Sprintf("%v,", i) - } - return r[:len(r)-1] -} - -const columnLabel = "col" - -func (ps *PQLToSQL) IssueQuery(req *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { - index := req.Index - q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() - if err != nil { - return nil, errors.Wrap(err, "PQLToSQL.IssueQuery parsing PQL") - } - var results pilosa.QueryResponse - for _, c := range q.Calls { - - if callIndex := c.CallIndex(); callIndex != "" { - index = callIndex - } - - // Handle the field arg. - switch strings.ToLower(c.Name) { - //handle the write queries - //TODO (twg) figure out sqlite changes() function to determine number of bits changed - case "set": - // Read colID. - colID, ok, err := c.UintArg("_" + columnLabel) - if err != nil { - return nil, fmt.Errorf("reading Set() column: %v", err) - } else if !ok { - return nil, fmt.Errorf("Set() column argument '%v' required", columnLabel) - } - ts, ok := c.Args["_timestamp"] - var timestamp string - if ok { - timestamp = strings.Replace(ts.(string), "T", " ", 1) - } - - field, err := c.FieldArg() - if err == nil { - rs, err := ps.Set(index, colID, field, c.Args[field], timestamp) - panicOn(err) - results.Results = append(results.Results, rs) - } else { - return nil, err - } - case "clear": - // Read colID. - colID, ok, err := c.UintArg("_" + columnLabel) - if err != nil { - return nil, fmt.Errorf("reading Clear() column: %v", err) - } else if !ok { - return nil, fmt.Errorf("Clear() column argument '%v' required", columnLabel) - } - - field, err := c.FieldArg() - if err == nil { - rs, err := ps.Clear(index, colID, field, c.Args[field]) - if err != nil { - return nil, fmt.Errorf("Clear() column: %v", err) - } - results.Results = append(results.Results, rs) - } else { - vv("c.FieldArg() returned '%v'", err) - } - case "clearrow": - field, err := c.FieldArg() - panicOn(err) - rs, err := ps.ClearRow(index, field, c.Args[field]) - panicOn(err) - results.Results = append(results.Results, rs) - case "store": - field, err := c.FieldArg() - panicOn(err) - rs, err := ps.Store(index, field, c.Args[field], c.Children) - panicOn(err) - results.Results = append(results.Results, rs) - default: //TODO (twg) move ToSQL into main package - rs, err := ps.readQuery(index, q) - panicOn(err) - results.Results = append(results.Results, rs) - } - } - return &results, nil -} -func (ps *PQLToSQL) handleBitmapCall(sql string) (*pilosa.Row, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - defer rows.Close() - columns := make([]uint64, 0) - for rows.Next() { - //really need a way to know what the results look like - var column uint64 - rows.Scan(&column) - columns = append(columns, column) - } - row := pilosa.NewRow(columns...) - return row, nil -} - -func (ps *PQLToSQL) handleIntCall(sql string) (count uint64, err error) { - rows, err := ps.DB.Query(sql) - if err != nil { - return 0, err - } - defer rows.Close() - for rows.Next() { - //really need a way to know what the results look like - rows.Scan(&count) - return - } - return -} - -type Table struct { -} - -func (ps *PQLToSQL) handleSignedRowCall(sql string) (pilosa.SignedRow, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - - var id uint64 - var pos []uint64 - var neg []uint64 - defer rows.Close() - for rows.Next() { - err := rows.Scan(&id) - if err != nil { - return pilosa.SignedRow{}, err - } - if id >= 0 { - pos = append(pos, id) - } else { - neg = append(neg, id) - } - } - err = rows.Err() - if err != nil { - return pilosa.SignedRow{}, err - } - posRow := pilosa.NewRow(pos...) - negRow := pilosa.NewRow(neg...) - return pilosa.SignedRow{Pos: posRow, Neg: negRow}, nil - -} -func (ps *PQLToSQL) handleStringArrayCall(sql string) ([]interface{}, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - rowKeys := make([]interface{}, 0) - var field string - var id string - defer rows.Close() - for rows.Next() { - err := rows.Scan(&field, &id) - if err != nil { - return nil, err - } - rowKeys = append(rowKeys, id) - } - err = rows.Err() - if err != nil { - return nil, err - } - return rowKeys, nil -} - -func (ps *PQLToSQL) handleIntArrayCall(sql string) (map[string][]interface{}, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - rowIDs := make([]interface{}, 0) - var id uint64 - var field string - defer rows.Close() - for rows.Next() { - err := rows.Scan(&field, &id) - if err != nil { - return nil, err - } - rowIDs = append(rowIDs, id) - } - err = rows.Err() - if err != nil { - return nil, err - } - m := map[string][]interface{}{"rows": rowIDs, "keys": []interface{}{}} - return m, nil -} -func (ps *PQLToSQL) handleGroupCountCall(sqlin string, index string, numFields int) (pilosa.SortedGroupCount, error) { - rows, err := ps.DB.Query(sqlin) - panicOn(err) - //groupCounts := make([]pilosa.GroupCount, 0) - var groupCounts pilosa.SortedGroupCount - defer rows.Close() - cols, err := rows.Columns() - if err != nil { - return nil, err - } - row := make([]interface{}, len(cols)) - for i := range cols { - row[i] = new(sql.RawBytes) - } - for rows.Next() { - // need a row*NumFields + cnt - //err := rows.Scan(&field, &row, &count) - groupCount := pilosa.GroupCount{} - - err := rows.Scan(row...) - if err != nil { - return nil, err - } - for i := 0; i < len(row)-1; i += 2 { - ptr := row[i].(*sql.RawBytes) - fieldName := string(*ptr) - fr := pilosa.FieldRow{Field: fieldName} - fi, err := ps.GetField(index, fieldName) - if err != nil { - return nil, err - } - ptr = row[i+1].(*sql.RawBytes) - v := string(*ptr) - if fi.Options.Keys { - fr.RowKey = v - - } else { - iv, err := strconv.Atoi(v) - if err != nil { - return nil, err - } - fr.RowID = uint64(iv) - } - - groupCount.Group = append(groupCount.Group, fr) - - } - ptr := row[len(row)-1].(*sql.RawBytes) - data := string(*ptr) //convert to string - i, err := strconv.ParseInt(data, 10, 64) - panicOn(err) - groupCount.Count = uint64(i) - groupCounts = append(groupCounts, groupCount) - - } - - err = rows.Err() - if err != nil { - return nil, err - } - return groupCounts, nil -} -func (ps *PQLToSQL) handlePairsFieldCall(sql, index string) (pilosa.PairsField, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - var count uint64 - var row string - defer rows.Close() - res := pilosa.PairsField{} - for rows.Next() { - err := rows.Scan(&row, &count) - if err != nil { - return pilosa.PairsField{}, err - } - pair := pilosa.Pair{Key: row, Count: count} - res.Pairs = append(res.Pairs, pair) - } - err = rows.Err() - if err != nil { - return pilosa.PairsField{}, err - } - return res, nil -} - -// special structures to deal with Extract. In the future I would like to use -// pilosa structs, but dealing with keys makes that complicated for now -// note the column is a string, but i only worry about the type when marshalling out for -// the sqlite part is typeless. Handling keys is almost seemless and not requiring translation -type extractColumns struct { - column string - rows map[int][]interface{} - parent *sauronExtractedIDMatrix -} - -func (eid extractColumns) MarshalJSON() ([]byte, error) { - buf := new(bytes.Buffer) - buf.WriteString(fmt.Sprintf(`{"column": %v,"rows":[`, eid.column)) - rows := make([][]interface{}, len(eid.parent.fields)) - for k, v := range eid.rows { - rows[k] = v - } - for i, grow := range rows { - if i != 0 { - buf.WriteByte(44) //write the comma - } - isKey := eid.parent.fields[i].isKey - buf.WriteString(`[`) - for d, row := range grow { - if d != 0 { - buf.WriteByte(44) //write the comma - } - if isKey { - buf.WriteString(fmt.Sprintf(`"%v"`, row)) - } else { - buf.WriteString(fmt.Sprintf(`%v`, row)) - } - - } - buf.WriteString(`]`) - } - - buf.WriteString(`]}`) - return buf.Bytes(), nil -} - -type extractFields struct { - Name string `json:"name"` - Type string `json:"type"` - isKey bool -} -type sauronExtractedIDMatrix struct { - columns []extractColumns - fields []extractFields -} - -func (eid sauronExtractedIDMatrix) MarshalJSON() ([]byte, error) { - buf := new(bytes.Buffer) - sort.Slice(eid.columns, func(i, j int) bool { - return eid.columns[i].column < eid.columns[j].column - }) - buf.WriteString(`{"columns":[`) - for i, col := range eid.columns { - b, err := col.MarshalJSON() - panicOn(err) - if i != 0 { - buf.WriteByte(44) //write the comma - } - buf.Write(b) - } - - buf.WriteString(`],"fields":[`) - for i, f := range eid.fields { - if i != 0 { - buf.WriteByte(44) - } - b, err := json.Marshal(f) - panicOn(err) - buf.Write(b) - } - buf.WriteString(`]}`) - return buf.Bytes(), nil -} - -func (ps *PQLToSQL) handleTableCall(sql, index string) (sauronExtractedIDMatrix, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - var field string - var row string - var column string - defer rows.Close() - col := make(map[string]map[string][]interface{}) - fidx := make(map[string]int) - results := sauronExtractedIDMatrix{} - counter := 0 - empty := make(map[string][]interface{}) - for rows.Next() { - err := rows.Scan(&field, &row, &column) - if err != nil { - return sauronExtractedIDMatrix{}, err - } - if field != "" { - _, ok := fidx[field] - if !ok { - fidx[field] = counter - counter++ - } - rows, ok := col[column] - if !ok { - rows = make(map[string][]interface{}) - } - rows[field] = append(rows[field], row) - //results.Columns = append(results.Columns, pilosa.ExtractedIDColumn{ColumnID: column, Rows: [][]uint64{}}) - col[column] = rows - } else { //constrow, or no filed info - col[column] = empty - } - } - err = rows.Err() - if err != nil { - return sauronExtractedIDMatrix{}, err - } - for c, rowMap := range col { - idc := extractColumns{rows: make(map[int][]interface{}), parent: &results} - idc.column = c - for fld, rows := range rowMap { - idx := fidx[fld] - idc.rows[idx] = rows - } - results.columns = append(results.columns, idc) - } - results.fields = make([]extractFields, len(fidx)) - for fieldName, idx := range fidx { - fi, _ := ps.GetField(index, fieldName) - // TODO (twg) hack need to figure out proper way to determine type - aType := "[]uint64" - if fi.Options.Keys { - aType = "[]string" - } - results.fields[idx] = extractFields{Name: fieldName, Type: aType, isKey: fi.Options.Keys} - } - - return results, nil -} - -func (ps *PQLToSQL) handleAggSQLCall(sql, index string) (pilosa.ValCount, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - defer rows.Close() - results := pilosa.ValCount{} - var val int64 - var count int64 - for rows.Next() { - err := rows.Scan(&val, &count) - if err != nil { - return pilosa.ValCount{}, err - } - results.Count = count - results.Val = val - break //shouldn't be anymore rows but being safe - } - err = rows.Err() - if err != nil { - return pilosa.ValCount{}, err - } - return results, nil -} -func (ps *PQLToSQL) handleBoolCall(sql, index string) (bool, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - defer rows.Close() - var val bool - for rows.Next() { - err := rows.Scan(&val) - if err != nil { - return false, err - } - break //shouldn't be anymore rows but being safe - } - err = rows.Err() - if err != nil { - return val, err - } - return val, nil -} - -func (ps *PQLToSQL) readQuery(index string, query *pql.Query) (interface{}, error) { - //Row(color=red) - sql, resultType, optional, err := ToSql(query.Calls, index) - panicOn(err) - switch resultType { - case BITMAP: - return ps.handleBitmapCall(sql) - case INT: - return ps.handleIntCall(sql) - case SIGNEDROW: - return ps.handleSignedRowCall(sql) - case ARRAY: - fieldName := optional[0].(string) - info, ok := ps.i2f[index] - if !ok { - return nil, errors.New(fmt.Sprintf("index not in schema %v", index)) - } - field, ok := info.Fields[fieldName] - if !ok { - return nil, errors.New(fmt.Sprintf("field not in schema %v", fieldName)) - } - if field.Options.Keys { - return ps.handleStringArrayCall(sql) - } else { - return ps.handleIntArrayCall(sql) - } - case GROUPCOUNT: - return ps.handleGroupCountCall(sql, index, optional[0].(int)) - case PAIRSFIELD: - return ps.handlePairsFieldCall(sql, index) - case TABLE: - return ps.handleTableCall(sql, index) - case AGGSQL: - return ps.handleAggSQLCall(sql, index) - case BOOL: - return ps.handleBoolCall(sql, index) - default: - vv("Unknown query Type %v", resultType) - } - return nil, nil -} - -type Int64Slice []int64 - -func (p Int64Slice) Len() int { return len(p) } -func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } -func (p Int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - -// on tables starting with ps.dbprefix will be returned. -func (ps *PQLToSQL) ListIndexFields() (tables []string) { - for indexName, index := range ps.i2f { - for fieldname, _ := range index.Fields { - - tables = append(tables, indexName+"/"+fieldname) - } - } - sort.Strings(tables) - return tables -} diff --git a/cmd/sauron/populate.sh b/cmd/sauron/populate.sh deleted file mode 100644 index 46e7dae67..000000000 --- a/cmd/sauron/populate.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -mod=20 -for i in `seq 100 1`;do -b=$(dc -e "$i $mod %p") -[[ $b == 0 ]] && mod=$(($mod+1)) -r=$(($b+30)) -payload="Set($i, j=$r)" -echo $payload -curl -XPOST "localhost:10101/index/i/query" -d "$payload" -done diff --git a/cmd/sauron/sample_schema.json b/cmd/sauron/sample_schema.json deleted file mode 100644 index 4dcc2c501..000000000 --- a/cmd/sauron/sample_schema.json +++ /dev/null @@ -1 +0,0 @@ -{"indexes":[{"name":"trait_store","createdAt":1595896639330112903,"options":{"keys":true,"trackExistence":true},"fields":[{"name":"aba","createdAt":1595896641512346986,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"ach_batch_recency","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_pass_thru_recency","createdAt":1595896640333428009,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_payment_recency","createdAt":1595896642230184052,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"bools","createdAt":1595896640151751600,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"bools-exists","createdAt":1595896640855732760,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"central_group","createdAt":1595896640036032355,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"custom_audiences","createdAt":1595896643284443107,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"days_since_last_logon","createdAt":1595896639332677618,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":1,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"db","createdAt":1595896639548206413,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"desktop_recency","createdAt":1595896642633713852,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"domestic_wire_recency","createdAt":1595896642264402807,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"external_transfer_recency","createdAt":1595896639332854139,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"fields","createdAt":1595896639552562813,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"funds_transfer_recency","createdAt":1595896640803192694,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"international_wire_recency","createdAt":1595896643511015744,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"mobile_remote_deposit_recency","createdAt":1595896641174868205,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"payroll_recency","createdAt":1595896642880416952,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"pfm_category_total_current_balance__401k_investment","createdAt":1595896641492935339,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__403b_investment","createdAt":1595896639553865549,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__529_investment","createdAt":1595896643478533291,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__any","createdAt":1595896642202343868,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__auto_loan","createdAt":1595896642260798767,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__brokerage_account","createdAt":1595896639550130078,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__certificate_of_deposit","createdAt":1595896643210723905,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__checking","createdAt":1595896640763548235,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":42,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__credit_card","createdAt":1595896639734800167,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__home_equity_loan","createdAt":1595896642644368353,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__ira_investment","createdAt":1595896642076795921,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__line_of_credit","createdAt":1595896642071432710,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__loan","createdAt":1595896643101755055,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__money_market","createdAt":1595896642574933677,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":36,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__mortgage","createdAt":1595896640934767155,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__personal_loan","createdAt":1595896641378825138,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__roth_ira_investment","createdAt":1595896642821349899,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__savings","createdAt":1595896639534597334,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__simple_ira","createdAt":1595896639952200646,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__student_loan","createdAt":1595896642329394180,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__taxable_investment","createdAt":1595896641406468398,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"phone_recency","createdAt":1595896639708002303,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_count__brokerage_account","createdAt":1595896642501159753,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_cd_or_share_certificate","createdAt":1595896643269910233,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_checking_or_share_draft","createdAt":1595896640044097611,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_credit_card","createdAt":1595896641500609762,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_indirect_auto_loan","createdAt":1595896643353298900,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_line_of_credit","createdAt":1595896641219946523,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_loan","createdAt":1595896641263557904,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_money_market","createdAt":1595896642506172994,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_new_auto_loan","createdAt":1595896640791087766,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_real_estate_loan","createdAt":1595896642616627035,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_savings_or_share","createdAt":1595896642313967229,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_credit_card","createdAt":1595896642254165546,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_loan","createdAt":1595896642870853670,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_checking_or_share_draft","createdAt":1595896639333642847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_credit_card","createdAt":1595896642761030299,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_savings_or_share","createdAt":1595896643227769780,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_vehicle_loan","createdAt":1595896642991171391,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_used_auto_loan","createdAt":1595896642865599530,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_basic_checking_or_share_draft","createdAt":1595896640110059946,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_cd_or_share_certificate","createdAt":1595896640174468431,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_credit_card","createdAt":1595896640192571502,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_indirect_auto_loan","createdAt":1595896640563441892,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896643543919820,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_line_of_credit","createdAt":1595896640188449923,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_loan","createdAt":1595896639761090179,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_money_market","createdAt":1595896642731261748,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_new_auto_loan","createdAt":1595896643143947051,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_checking_or_share_draft","createdAt":1595896640346982139,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_savings_or_share","createdAt":1595896641346726397,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_real_estate_loan","createdAt":1595896643351578694,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_checking_or_share_draft","createdAt":1595896640302899735,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_savings_or_share","createdAt":1595896642726551847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_savings_or_share","createdAt":1595896642961872488,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_credit_card","createdAt":1595896640022927417,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_loan","createdAt":1595896642352438551,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_checking_or_share_draft","createdAt":1595896640804808345,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_credit_card","createdAt":1595896641113696262,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_savings_or_share","createdAt":1595896640312498993,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_vehicle_loan","createdAt":1595896639886308884,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_used_auto_loan","createdAt":1595896642412302198,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__debit_card","createdAt":1595896643127876947,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__education_savings_or_share","createdAt":1595896639338093281,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__escrow_account","createdAt":1595896642969874721,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__health_savings_account","createdAt":1595896640570458731,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_line_of_credit","createdAt":1595896642592520668,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_loan","createdAt":1595896643091627581,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__insurance_product","createdAt":1595896639744476133,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__interest_only_legal_trust_account","createdAt":1595896643005843869,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__membership_share","createdAt":1595896641732585653,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__other_service","createdAt":1595896641807551269,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_401k","createdAt":1595896639333271302,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":11,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_403b","createdAt":1595896643165638942,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_529","createdAt":1595896641640608805,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira","createdAt":1595896639555618615,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira_cd_or_share_certificate","createdAt":1595896641504460031,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__small_business_checking_or_share_draft","createdAt":1595896639332189957,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__special_indirect_vehicle_loan","createdAt":1595896641880199490,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__student_education_loan","createdAt":1595896639551558423,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira","createdAt":1595896643372890708,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira_cd_or_share_certificate","createdAt":1595896641621211665,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__trust","createdAt":1595896640709548779,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__vacation_club_holiday_savings","createdAt":1595896642256997553,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_recency__brokerage_account","createdAt":1595896641955724697,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_cd_or_share_certificate","createdAt":1595896642377298176,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_checking_or_share_draft","createdAt":1595896641640736297,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_credit_card","createdAt":1595896641199968081,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_indirect_auto_loan","createdAt":1595896643314414987,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_line_of_credit","createdAt":1595896641576267257,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_loan","createdAt":1595896641355763147,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_money_market","createdAt":1595896639970740640,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_new_auto_loan","createdAt":1595896642844117593,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_real_estate_loan","createdAt":1595896642953871689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_savings_or_share","createdAt":1595896643325231815,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_credit_card","createdAt":1595896642698454456,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_loan","createdAt":1595896641951407002,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_checking_or_share_draft","createdAt":1595896642711316173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_credit_card","createdAt":1595896642147545663,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_savings_or_share","createdAt":1595896642364919307,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_vehicle_loan","createdAt":1595896639876538030,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_used_auto_loan","createdAt":1595896643527381053,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_basic_checking_or_share_draft","createdAt":1595896639874545938,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_cd_or_share_certificate","createdAt":1595896643147292944,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_credit_card","createdAt":1595896642605033674,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_indirect_auto_loan","createdAt":1595896641097157897,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896642523867033,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_line_of_credit","createdAt":1595896641979726767,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_loan","createdAt":1595896639741458231,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_money_market","createdAt":1595896642350195677,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_new_auto_loan","createdAt":1595896642852653224,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_checking_or_share_draft","createdAt":1595896641391887646,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_savings_or_share","createdAt":1595896643501816261,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_real_estate_loan","createdAt":1595896643015250459,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_checking_or_share_draft","createdAt":1595896640413965217,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_savings_or_share","createdAt":1595896642440765022,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_savings_or_share","createdAt":1595896643523788186,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_credit_card","createdAt":1595896641228514181,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_loan","createdAt":1595896642815200189,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_checking_or_share_draft","createdAt":1595896639648900382,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_credit_card","createdAt":1595896639596071703,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_savings_or_share","createdAt":1595896642447703684,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_vehicle_loan","createdAt":1595896639863224318,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_used_auto_loan","createdAt":1595896639337315659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__debit_card","createdAt":1595896641091084110,"options":{"type":"int","base":0,"bitDepth":1,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__education_savings_or_share","createdAt":1595896643243122929,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__escrow_account","createdAt":1595896642288649376,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__health_savings_account","createdAt":1595896639713687734,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_line_of_credit","createdAt":1595896640644328270,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_loan","createdAt":1595896643005117660,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__insurance_product","createdAt":1595896640080338115,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__interest_only_legal_trust_account","createdAt":1595896640710544689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__membership_share","createdAt":1595896640920327282,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__other_service","createdAt":1595896641868683380,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_401k","createdAt":1595896643214447056,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_403b","createdAt":1595896639331510173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_529","createdAt":1595896641093053659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira","createdAt":1595896640406186395,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira_cd_or_share_certificate","createdAt":1595896642473602012,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__small_business_checking_or_share_draft","createdAt":1595896640631139451,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__special_indirect_vehicle_loan","createdAt":1595896641787887178,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__student_education_loan","createdAt":1595896640496522510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira","createdAt":1595896643383850510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira_cd_or_share_certificate","createdAt":1595896642145329304,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__trust","createdAt":1595896640506789797,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__vacation_club_holiday_savings","createdAt":1595896642206110046,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"stop_payment_recency","createdAt":1595896640954094741,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"survey_2053c210_adfc_4e14_afd3_9abc68a70719","createdAt":1595896642841258189,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2679bd7e_cdb8_4c24_a1da_e904799382eb","createdAt":1595896642498273724,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2f336120_7130_492b_b116_d93d560baa0a","createdAt":1595896639333711539,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_36e229ff_c0a6_48ad_928e_19b5abf69f11","createdAt":1595896642987004213,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa","createdAt":1595896643012024691,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796","createdAt":1595896640184709609,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_587ae933_b8e3_49a2_a23d_989df9ad119f","createdAt":1595896642862965911,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab","createdAt":1595896642292355510,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949","createdAt":1595896640788795677,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_9cebb459_162e_4518_88f6_bc1cdd630cd0","createdAt":1595896642343098690,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_a05de6d3_6620_4eb2_b9a2_511936680b57","createdAt":1595896643389135517,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_b570363c_ba94_4a7a_9891_141a1befc42b","createdAt":1595896643120229702,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_c358d6e0_4116_4230_84ed_869b315e89a1","createdAt":1595896642202109079,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_cce773ce_14e8_4fae_90cd_213a821a3bc0","createdAt":1595896640040535589,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_e49df858_4903_40ac_bd9f_96f764940f2c","createdAt":1595896642346035396,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9","createdAt":1595896640643626147,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"tablet_recency","createdAt":1595896640414042095,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"user_id","createdAt":1595896640846590883,"options":{"type":"int","base":0,"bitDepth":21,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"zip_code","createdAt":1595896640534832822,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}}],"shardWidth":1048576}]} diff --git a/cmd/sauron/sauron_test.go b/cmd/sauron/sauron_test.go deleted file mode 100644 index 53cf786de..000000000 --- a/cmd/sauron/sauron_test.go +++ /dev/null @@ -1,396 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "encoding/hex" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strconv" - "testing" - - pb "github.com/golang/protobuf/proto" //nolint:staticcheck - pbuf "github.com/molecula/go-pilosa/v2/gopilosa_pbuf" - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/encoding/proto" - "github.com/pkg/errors" -) - -func TestSauron(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //remove the existing db file if present - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - - schemaJson := `{"indexes":[{"name":"scratch","createdAt":1611185870149882000,"options":{"keys":false,"trackExistence":true}, "fields":[{"name":"luminosity","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}}]}]}` - - client := &http.Client{} - t.Run("Schema", func(t *testing.T) { - _, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) - panicOn(err) - // verify that we can fetch it back - resp, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp.Body) - panicOn(err) - sbody := string(body) - if sbody[:62] != schemaJson[:62] { - fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) - panic("unexpected schema back") - } - }) - // add a set field "color" - //POST localhost:10101/index/scratch/field/color - t.Run("Create Non-Keyed SetField", func(t *testing.T) { - resp, err := client.Post(url+"/index/scratch/field/rating", "application/text", bytes.NewBuffer(nil)) - panicOn(err) - if resp.StatusCode != 200 { - panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) - } - }) - t.Run("Create Keyed SetField", func(t *testing.T) { - resp, err := client.Post(url+"/index/scratch/field/color", "application/text", bytes.NewBuffer([]byte(`{"options": {"keys": true}}`))) - panicOn(err) - if resp.StatusCode != 200 { - panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) - } - }) - t.Run("Create Time Field", func(t *testing.T) { - resp, err := client.Post(url+"/index/scratch/field/event", "application/text", bytes.NewBuffer([]byte(`{"options": { "type": "time", "timeQuantum": "YMDH" }}`))) - panicOn(err) - if resp.StatusCode != 200 { - panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) - } - }) - - // set some bits - - // set some bits - - t.Run("Set", func(t *testing.T) { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) - panicOn(err) - // test for dups - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) - panicOn(err) - // - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=red)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=green)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=green)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=green)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=green)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=yellow)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=yellow)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=yellow)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=yellow)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=orange)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=orange)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=orange)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=orange)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, luminosity=1)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, luminosity=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, luminosity=4)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, luminosity=5)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, luminosity=4)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, luminosity=3)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, luminosity=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, luminosity=1)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, rating=1)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(10, rating=1)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, rating=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, rating=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, rating=3)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, rating=4)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, rating=4)`)) - panicOn(err) - - //time fields - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T01:00)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T02:00)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, event=1,2021-02-05T02:05)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, event=1,2021-02-05T03:00)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, event=1,2021-02-05T04:00)`)) - panicOn(err) - }) - t.Run("Clear", func(t *testing.T) { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Clear(1, color=purple)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`ClearRow(color=purple)`)) - panicOn(err) - }) - t.Run("Store", func(t *testing.T) { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Store(Row(color=red), color=purple)`)) - panicOn(err) - }) - // verify that we can fetch it back - var schema pilosa.Schema - t.Run("Fetch Schema", func(t *testing.T) { - resp, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp.Body) - panicOn(err) - err = json.Unmarshal(body, &schema) - if err != nil { - panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) - } - }) - t.Run("Import Roaring", func(t *testing.T) { - //sets 12 bits in shard 0 - roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - idx := schema.Indexes[0] - var fld *pilosa.FieldInfo - for _, f := range idx.Fields { - if f.Name == "rating" { - fld = f - } - } - if fld == nil { - panic("field color not found") - } - msg := pilosa.ImportRoaringRequest{ - IndexCreatedAt: idx.CreatedAt, - FieldCreatedAt: fld.CreatedAt, - Clear: false, - Views: map[string][]byte{ - "": roaringData, - }, - UpdateExistence: true, - } - ser := proto.Serializer{} - data, err := ser.Marshal(&msg) - if err != nil { - t.Fatal(err) - } - furl := url + "/index/scratch/field/rating/import-roaring/0" - req, err := http.NewRequest("POST", furl, bytes.NewBuffer(data)) - panicOn(err) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - - // maybe need to Accept: "application/x-protobuf" - resp, err := client.Do(req) - panicOn(err) - _ = resp - }) - t.Run("Read Query", func(t *testing.T) { - for _, tst := range GetTests() { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(tst.Pql)) - panicOn(err) - } - }) -} - -func makeImportRequest(field *FieldInfo2, shard uint64, rows, columns []uint64, clear bool) (path string, data []byte, err error) { - msg := &pbuf.ImportRequest{ - Index: field.GetIndexName(), - IndexCreatedAt: field.GetIndexCreatedAt(), - Field: field.GetName(), - FieldCreatedAt: field.GetCreatedAt(), - Shard: shard, - RowIDs: rows, - ColumnIDs: columns, - } - data, err = pb.Marshal(msg) - if err != nil { - return "", nil, errors.Wrap(err, "marshaling Import to protobuf") - } - path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) - return path, data, nil -} -func makeImportKeyRequest(field *FieldInfo2, shard uint64, rows []string, columns []uint64, clear bool) (path string, data []byte, err error) { - msg := &pbuf.ImportRequest{ - Index: field.GetIndexName(), - IndexCreatedAt: field.GetIndexCreatedAt(), - Field: field.GetName(), - FieldCreatedAt: field.GetCreatedAt(), - Shard: shard, - RowKeys: rows, - ColumnIDs: columns, - } - data, err = pb.Marshal(msg) - if err != nil { - return "", nil, errors.Wrap(err, "marshaling Import to protobuf") - } - path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) - return path, data, nil -} - -func makeImportValueRequest(field *FieldInfo2, shard uint64, values []int64, columns []uint64, clear bool) (path string, data []byte, err error) { - msg := &pbuf.ImportValueRequest{ - Index: field.GetIndexName(), - IndexCreatedAt: field.GetIndexCreatedAt(), - Field: field.GetName(), - FieldCreatedAt: field.GetCreatedAt(), - Shard: shard, - Values: values, - ColumnIDs: columns, - } - data, err = pb.Marshal(msg) - if err != nil { - return "", nil, errors.Wrap(err, "marshaling Import to protobuf") - } - path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) - return path, data, nil -} -func TestSauronImport(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //remove the existing db file if present - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - schemaJson := `{ - "indexes": [{ - "name": "scratch", - "createdAt": 1611185870149882000, - "options": { - "keys": false, - "trackExistence": true - }, - "fields": [{ - "name": "bsi", - "createdAt": 1595896639332730413, - "options": { - "type": "int", - "base": 0, - "bitDepth": 31, - "min": -9223372036854775808, - "max": 9223372036854775807, - "keys": false, - "foreignIndex": "" - } - }, { - "name": "decimal", - "createdAt": 1613177295654228860, - "options": { - "type": "decimal", - "base": 0, - "scale": 1, - "bitDepth": 0, - "min": -922337203685477580.8, - "max": 922337203685477580.7, - "keys": false - } - },{ - "name": "setkeyfield", - "createdAt": 1613345793598357200, - "options": { - "type": "set", - "cacheType": "ranked", - "cacheSize": 50000, - "keys": true - } - },{ - "name": "setfield", - "createdAt": 1613345793598357200, - "options": { - "type": "set", - "cacheType": "ranked", - "cacheSize": 50000, - "keys": false - } - }] - }] -}` - client := &http.Client{} - _, err = client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) - panicOn(err) - - t.Run("basic row/column", func(t *testing.T) { - f, err := proxy.sql.GetField("scratch", "setfield") - panicOn(err) - clear := false - cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} - rows := []uint64{0, 0, 0, 1, 1, 1, 2, 2, 2} - shard := uint64(0) - path, payload, err := makeImportRequest(f, shard, rows, cols, clear) - _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) - panicOn(err) - }) - t.Run("basic rowkey/column", func(t *testing.T) { - f, err := proxy.sql.GetField("scratch", "setkeyfield") - panicOn(err) - clear := false - cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} - rows := []string{"red", "red", "red", "blue", "blue", "blue", "green", "green", "green"} - shard := uint64(0) - path, payload, err := makeImportKeyRequest(f, shard, rows, cols, clear) - _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) - panicOn(err) - }) - t.Run("basic import values", func(t *testing.T) { - f, err := proxy.sql.GetField("scratch", "bsi") - panicOn(err) - clear := false - cols := []uint64{0, 1, 2, 3, 4} - values := []int64{40, 30, 20, 10, 5} - shard := uint64(0) - path, payload, err := makeImportValueRequest(f, shard, values, cols, clear) - _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) - panicOn(err) - }) - -} diff --git a/cmd/sauron/schema_test.go b/cmd/sauron/schema_test.go deleted file mode 100644 index 23f5e924b..000000000 --- a/cmd/sauron/schema_test.go +++ /dev/null @@ -1,353 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strings" - "testing" - - "github.com/pilosa/pilosa/v2" -) - -func NewSauronTestConfig(t *testing.T) *SauronConfig { - - bind := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) - bindGRPC := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) - gossipPort := fmt.Sprintf("%d", GetAvailPort()) - dataDir, err := ioutil.TempDir("", "pilosa-sauron-target-*") - panicOn(err) - - return &SauronConfig{ - DatabaseName: strings.ToLower(t.Name()), - Bind: bind, - BindGRPC: bindGRPC, - GossipPort: gossipPort, - DataDir: dataDir, - } -} - -func TestProxyCanHandlePostSchema(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //start with - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - - // prep for test by cleaning up leftovers from any old run. - //panicOn(proxy.sql.DropTablesWithPrefix(t.Name())) - - //schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } }, { "name ": "color", "options ": {} } ] }] }` - /* - schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } } ] }] }` - - - vv("Posting err = '%v'", schemaJson) - resp, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) - if resp.StatusCode != http.StatusNoContent { - panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) - } - // verify that we can fetch it back - resp2, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp2.Body) - panicOn(err) - sbody := string(body) - vv("body = '%v'", sbody) - if sbody[:62] != schemaJson[:62] { - fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) - panic("unxpected schema back") - } - */ - // load large schema too - client := &http.Client{} - schm, err := ioutil.ReadFile("sample_schema.json") - panicOn(err) - - resp, err := client.Post(url+"/schema", "application/json", bytes.NewBuffer(schm)) - panicOn(err) - if resp.StatusCode != http.StatusNoContent { - panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) - } - - // verify all tables expected are present - obs := make(map[string]bool) - for i, table := range proxy.sql.ListIndexFields() { - _ = i - obs[table] = true - if !expectedTables[table] { - panic(fmt.Sprintf("observed table '%v' but was not expected", table)) - } - } - for table := range expectedTables { - if !obs[table] { - panic(fmt.Sprintf("expected table '%v' but was not observed", table)) - } - } - -} - -func TestProxyCanCreateIndexFieldViaPost(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //start with - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - client := &http.Client{} - //create non-keyed or "simple" index - resp, err := client.Post(url+"/index/simple", "application/json", bytes.NewBuffer(nil)) - panicOn(err) - if resp.StatusCode != http.StatusOK { - panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) - } - - fields := []struct { - name string - options []byte - }{ - {name: "set", options: []byte{}}, - {name: "keyset", options: []byte{}}, - {name: "bsi", options: []byte{}}, - } - for _, field := range fields { - //create set field - resp, err = client.Post(fmt.Sprintf("%v/index/simple/field/%v", url, field.name), "application/json", bytes.NewBuffer(field.options)) - panicOn(err) - if resp.StatusCode != http.StatusOK { - panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) - } - } - // verify that we can fetch it back - resp2, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp2.Body) - panicOn(err) - var schema pilosa.Schema - err = json.Unmarshal(body, &schema) - if err != nil { - panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) - } - - expected := map[string]map[string]bool{ - "simple": {"bsi": true, "set": true, "keyset": true}, - } - for _, i := range schema.Indexes { - ll, ok := expected[i.Name] - if !ok { - panic(fmt.Sprintf("expected index not present: %v", i.Name)) - } - for _, f := range i.Fields { - _, ok := ll[f.Name] - if !ok { - panic(fmt.Sprintf("expected field '%v' not present in index'%v'", f.Name, i.Name)) - } - } - } -} - -var expectedTables = map[string]bool{ - "trait_store/aba": true, - "trait_store/ach_batch_recency": true, - "trait_store/ach_pass_thru_recency": true, - "trait_store/ach_payment_recency": true, - "trait_store/bools": true, - "trait_store/bools-exists": true, - "trait_store/central_group": true, - "trait_store/custom_audiences": true, - "trait_store/days_since_last_logon": true, - "trait_store/db": true, - "trait_store/desktop_recency": true, - "trait_store/domestic_wire_recency": true, - "trait_store/external_transfer_recency": true, - "trait_store/fields": true, - "trait_store/funds_transfer_recency": true, - "trait_store/international_wire_recency": true, - "trait_store/mobile_remote_deposit_recency": true, - "trait_store/payroll_recency": true, - "trait_store/pfm_category_total_current_balance__401k_investment": true, - "trait_store/pfm_category_total_current_balance__403b_investment": true, - "trait_store/pfm_category_total_current_balance__529_investment": true, - "trait_store/pfm_category_total_current_balance__any": true, - "trait_store/pfm_category_total_current_balance__auto_loan": true, - "trait_store/pfm_category_total_current_balance__brokerage_account": true, - "trait_store/pfm_category_total_current_balance__certificate_of_deposit": true, - "trait_store/pfm_category_total_current_balance__checking": true, - "trait_store/pfm_category_total_current_balance__credit_card": true, - "trait_store/pfm_category_total_current_balance__home_equity_loan": true, - "trait_store/pfm_category_total_current_balance__ira_investment": true, - "trait_store/pfm_category_total_current_balance__line_of_credit": true, - "trait_store/pfm_category_total_current_balance__loan": true, - "trait_store/pfm_category_total_current_balance__money_market": true, - "trait_store/pfm_category_total_current_balance__mortgage": true, - "trait_store/pfm_category_total_current_balance__personal_loan": true, - "trait_store/pfm_category_total_current_balance__roth_ira_investment": true, - "trait_store/pfm_category_total_current_balance__savings": true, - "trait_store/pfm_category_total_current_balance__simple_ira": true, - "trait_store/pfm_category_total_current_balance__student_loan": true, - "trait_store/pfm_category_total_current_balance__taxable_investment": true, - "trait_store/phone_recency": true, - "trait_store/product_count__brokerage_account": true, - "trait_store/product_count__commercial_cd_or_share_certificate": true, - "trait_store/product_count__commercial_checking_or_share_draft": true, - "trait_store/product_count__commercial_credit_card": true, - "trait_store/product_count__commercial_indirect_auto_loan": true, - "trait_store/product_count__commercial_line_of_credit": true, - "trait_store/product_count__commercial_loan": true, - "trait_store/product_count__commercial_money_market": true, - "trait_store/product_count__commercial_new_auto_loan": true, - "trait_store/product_count__commercial_real_estate_loan": true, - "trait_store/product_count__commercial_savings_or_share": true, - "trait_store/product_count__commercial_secured_credit_card": true, - "trait_store/product_count__commercial_secured_loan": true, - "trait_store/product_count__commercial_special_checking_or_share_draft": true, - "trait_store/product_count__commercial_special_credit_card": true, - "trait_store/product_count__commercial_special_savings_or_share": true, - "trait_store/product_count__commercial_special_vehicle_loan": true, - "trait_store/product_count__commercial_used_auto_loan": true, - "trait_store/product_count__consumer_basic_checking_or_share_draft": true, - "trait_store/product_count__consumer_cd_or_share_certificate": true, - "trait_store/product_count__consumer_credit_card": true, - "trait_store/product_count__consumer_indirect_auto_loan": true, - "trait_store/product_count__consumer_interest_bearing_checking_or_share_draft": true, - "trait_store/product_count__consumer_line_of_credit": true, - "trait_store/product_count__consumer_loan": true, - "trait_store/product_count__consumer_money_market": true, - "trait_store/product_count__consumer_new_auto_loan": true, - "trait_store/product_count__consumer_premium_checking_or_share_draft": true, - "trait_store/product_count__consumer_premium_savings_or_share": true, - "trait_store/product_count__consumer_real_estate_loan": true, - "trait_store/product_count__consumer_restricted_checking_or_share_draft": true, - "trait_store/product_count__consumer_restricted_savings_or_share": true, - "trait_store/product_count__consumer_savings_or_share": true, - "trait_store/product_count__consumer_secured_credit_card": true, - "trait_store/product_count__consumer_secured_loan": true, - "trait_store/product_count__consumer_special_checking_or_share_draft": true, - "trait_store/product_count__consumer_special_credit_card": true, - "trait_store/product_count__consumer_special_savings_or_share": true, - "trait_store/product_count__consumer_special_vehicle_loan": true, - "trait_store/product_count__consumer_used_auto_loan": true, - "trait_store/product_count__debit_card": true, - "trait_store/product_count__education_savings_or_share": true, - "trait_store/product_count__escrow_account": true, - "trait_store/product_count__health_savings_account": true, - "trait_store/product_count__home_equity_line_of_credit": true, - "trait_store/product_count__home_equity_loan": true, - "trait_store/product_count__insurance_product": true, - "trait_store/product_count__interest_only_legal_trust_account": true, - "trait_store/product_count__membership_share": true, - "trait_store/product_count__other_service": true, - "trait_store/product_count__plan_401k": true, - "trait_store/product_count__plan_403b": true, - "trait_store/product_count__plan_529": true, - "trait_store/product_count__roth_ira": true, - "trait_store/product_count__roth_ira_cd_or_share_certificate": true, - "trait_store/product_count__small_business_checking_or_share_draft": true, - "trait_store/product_count__special_indirect_vehicle_loan": true, - "trait_store/product_count__student_education_loan": true, - "trait_store/product_count__traditional_ira": true, - "trait_store/product_count__traditional_ira_cd_or_share_certificate": true, - "trait_store/product_count__trust": true, - "trait_store/product_count__vacation_club_holiday_savings": true, - "trait_store/product_recency__brokerage_account": true, - "trait_store/product_recency__commercial_cd_or_share_certificate": true, - "trait_store/product_recency__commercial_checking_or_share_draft": true, - "trait_store/product_recency__commercial_credit_card": true, - "trait_store/product_recency__commercial_indirect_auto_loan": true, - "trait_store/product_recency__commercial_line_of_credit": true, - "trait_store/product_recency__commercial_loan": true, - "trait_store/product_recency__commercial_money_market": true, - "trait_store/product_recency__commercial_new_auto_loan": true, - "trait_store/product_recency__commercial_real_estate_loan": true, - "trait_store/product_recency__commercial_savings_or_share": true, - "trait_store/product_recency__commercial_secured_credit_card": true, - "trait_store/product_recency__commercial_secured_loan": true, - "trait_store/product_recency__commercial_special_checking_or_share_draft": true, - "trait_store/product_recency__commercial_special_credit_card": true, - "trait_store/product_recency__commercial_special_savings_or_share": true, - "trait_store/product_recency__commercial_special_vehicle_loan": true, - "trait_store/product_recency__commercial_used_auto_loan": true, - "trait_store/product_recency__consumer_basic_checking_or_share_draft": true, - "trait_store/product_recency__consumer_cd_or_share_certificate": true, - "trait_store/product_recency__consumer_credit_card": true, - "trait_store/product_recency__consumer_indirect_auto_loan": true, - "trait_store/product_recency__consumer_interest_bearing_checking_or_share_draft": true, - "trait_store/product_recency__consumer_line_of_credit": true, - "trait_store/product_recency__consumer_loan": true, - "trait_store/product_recency__consumer_money_market": true, - "trait_store/product_recency__consumer_new_auto_loan": true, - "trait_store/product_recency__consumer_premium_checking_or_share_draft": true, - "trait_store/product_recency__consumer_premium_savings_or_share": true, - "trait_store/product_recency__consumer_real_estate_loan": true, - "trait_store/product_recency__consumer_restricted_checking_or_share_draft": true, - "trait_store/product_recency__consumer_restricted_savings_or_share": true, - "trait_store/product_recency__consumer_savings_or_share": true, - "trait_store/product_recency__consumer_secured_credit_card": true, - "trait_store/product_recency__consumer_secured_loan": true, - "trait_store/product_recency__consumer_special_checking_or_share_draft": true, - "trait_store/product_recency__consumer_special_credit_card": true, - "trait_store/product_recency__consumer_special_savings_or_share": true, - "trait_store/product_recency__consumer_special_vehicle_loan": true, - "trait_store/product_recency__consumer_used_auto_loan": true, - "trait_store/product_recency__debit_card": true, - "trait_store/product_recency__education_savings_or_share": true, - "trait_store/product_recency__escrow_account": true, - "trait_store/product_recency__health_savings_account": true, - "trait_store/product_recency__home_equity_line_of_credit": true, - "trait_store/product_recency__home_equity_loan": true, - "trait_store/product_recency__insurance_product": true, - "trait_store/product_recency__interest_only_legal_trust_account": true, - "trait_store/product_recency__membership_share": true, - "trait_store/product_recency__other_service": true, - "trait_store/product_recency__plan_401k": true, - "trait_store/product_recency__plan_403b": true, - "trait_store/product_recency__plan_529": true, - "trait_store/product_recency__roth_ira": true, - "trait_store/product_recency__roth_ira_cd_or_share_certificate": true, - "trait_store/product_recency__small_business_checking_or_share_draft": true, - "trait_store/product_recency__special_indirect_vehicle_loan": true, - "trait_store/product_recency__student_education_loan": true, - "trait_store/product_recency__traditional_ira": true, - "trait_store/product_recency__traditional_ira_cd_or_share_certificate": true, - "trait_store/product_recency__trust": true, - "trait_store/product_recency__vacation_club_holiday_savings": true, - "trait_store/stop_payment_recency": true, - "trait_store/survey_2053c210_adfc_4e14_afd3_9abc68a70719": true, - "trait_store/survey_2679bd7e_cdb8_4c24_a1da_e904799382eb": true, - "trait_store/survey_2f336120_7130_492b_b116_d93d560baa0a": true, - "trait_store/survey_36e229ff_c0a6_48ad_928e_19b5abf69f11": true, - "trait_store/survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa": true, - "trait_store/survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796": true, - "trait_store/survey_587ae933_b8e3_49a2_a23d_989df9ad119f": true, - "trait_store/survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab": true, - "trait_store/survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949": true, - "trait_store/survey_9cebb459_162e_4518_88f6_bc1cdd630cd0": true, - "trait_store/survey_a05de6d3_6620_4eb2_b9a2_511936680b57": true, - "trait_store/survey_b570363c_ba94_4a7a_9891_141a1befc42b": true, - "trait_store/survey_c358d6e0_4116_4230_84ed_869b315e89a1": true, - "trait_store/survey_cce773ce_14e8_4fae_90cd_213a821a3bc0": true, - "trait_store/survey_e49df858_4903_40ac_bd9f_96f764940f2c": true, - "trait_store/survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9": true, - "trait_store/tablet_recency": true, - "trait_store/user_id": true, - "trait_store/zip_code": true, -} diff --git a/cmd/sauron/sqlgen_test.go b/cmd/sauron/sqlgen_test.go deleted file mode 100644 index b9d11a728..000000000 --- a/cmd/sauron/sqlgen_test.go +++ /dev/null @@ -1,260 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/v2/pql" - "github.com/shurcooL/go-goon" -) - -/* -sqlite db this test validated this against -create table bits (field ,row , column , timestamp); - create table λbsi (field string,column bigint , val bigint); - CREATE VIEW columns as - select distinct column from λbits - UNION - select distinct column from λbsi; - - INSERT INTO λbits VALUES( 'color','red',1); - INSERT INTO λbits VALUES( 'color','red',2); - INSERT INTO λbits VALUES( 'color','red',3); - INSERT INTO λbits VALUES( 'color','red',4); - INSERT INTO λbits VALUES( 'color','red',5); - INSERT INTO λbits VALUES( 'color','green',2); - INSERT INTO λbits VALUES( 'color','green',4); - INSERT INTO λbits VALUES( 'color','green',6); - INSERT INTO λbits VALUES( 'color','green',8); - INSERT INTO λbits VALUES( 'color','yellow',1); - INSERT INTO λbits VALUES( 'color','yellow',3); - INSERT INTO λbits VALUES( 'color','yellow',5); - INSERT INTO λbits VALUES( 'color','yellow',7); - INSERT INTO λbits VALUES( 'color','orange',5); - INSERT INTO λbits VALUES( 'color','orange',6); - INSERT INTO λbits VALUES( 'color','orange',7); - INSERT INTO λbits VALUES( 'color','orange',8); - - INSERT INTO λbits VALUES( 'rating',1,8); - INSERT INTO λbits VALUES( 'rating',1,7); - INSERT INTO λbits VALUES( 'rating',1,10); - INSERT INTO λbits VALUES( 'rating',2,6); - INSERT INTO λbits VALUES( 'rating',2,5); - INSERT INTO λbits VALUES( 'rating',3,4); - INSERT INTO λbits VALUES( 'rating',4,3); - INSERT INTO λbits VALUES( 'rating',4,2); - - INSERT INTO λbsi VALUES( 'luminosity',1, 1); - INSERT INTO λbsi VALUES( 'luminosity',2, 2); - INSERT INTO λbsi VALUES( 'luminosity',3, 4); - INSERT INTO λbsi VALUES( 'luminosity',4, 5); - INSERT INTO λbsi VALUES( 'luminosity',5, 4); - INSERT INTO λbsi VALUES( 'luminosity',6, 3); - INSERT INTO λbsi VALUES( 'luminosity',7, 2); - INSERT INTO λbsi VALUES( 'luminosity',8, 1); -*/ -func GetTests() []struct { - Pql string - Sql string -} { - return []struct { - Pql string - Sql string - }{ - { - Pql: "Row(rating=0)", - Sql: `select distinct column from λbits where field="rating" AND row="0"`, - }, - { - Pql: "Row(color=red)", - Sql: `select distinct column from λbits where field="color" AND row="red"`, - }, - { - Pql: "Row(luminosity>2)", - Sql: `select distinct column from λbsi where field="luminosity" and val>2`, - }, - { - Pql: "Row(luminosity<2)", - Sql: `select distinct column from λbsi where field="luminosity" and val<2`, - }, - { - Pql: "Row(luminosity==4)", - Sql: `select distinct column from λbsi where field="luminosity" and val=4`, - }, - { - Pql: "Row(luminosity<=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val<=4`, - }, - { - Pql: "Row(luminosity>=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val>=4`, - }, - { - Pql: "Row(luminosity!=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val!=4`, - }, - { - Pql: "Row(2<=luminosity<=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val>=2 and val<=4`, - }, - { - Pql: "Row(22 and val<=4`, - }, - { - Pql: "Row(22 and val<4`, - }, - { - Pql: `Row(event=1, from='2021-02-01T00:00', to='2021-02-05T03:00')`, - Sql: `select distinct column from λbits where field="event" AND row="1" AND timestamp >= "2021-02-01 00:00" AND timestamp < "2021-02-05 03:00"`, - }, - { - Pql: "Count(row(color=red))", - Sql: `select count(*) from( select distinct column from λbits where field="color" AND row="red" )`, - }, - { - Pql: `Intersect(Row(color=red),Row(color=green))`, - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: "Count(Union(Row(color=red),Row(color=green)))", - Sql: `select count(*) from( select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") )`, - }, - { - Pql: "Union(Row(color=red),Row(color=green))", - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: "Difference(Row(color=red),Row(color=green))", - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" except select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: `All()`, - Sql: `select column from λcolumns`, - }, - { - Pql: "Not(Row(color=red))", - Sql: `select column from λcolumns except select distinct column from λbits where field="color" AND row="red"`, - }, - { - Pql: "Xor(Row(color=red),Row(color=green))", - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") except select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: "Distinct(field=luminosity)", - Sql: `select distinct val from λbsi where field="luminosity"`, - }, - { - Pql: "Distinct(Row(color=red),field=luminosity)", - Sql: `select distinct val from λbsi where field="luminosity" AND column in (select distinct column from λbits where field="color" AND row="red")`, - }, - { - Pql: "Distinct(Intersect(Row(color=red),Row(color=green)),field=luminosity)", - Sql: `select distinct val from λbsi where field="luminosity" AND column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green"))`, - }, - { - Pql: "Rows(color)", - Sql: `select distinct field, row from λbits where field="color" order by row`, - }, - { - Pql: "Rows(color, limit=10)", - Sql: `select distinct field, row from λbits where field="color" order by row limit 10`, - }, - { - Pql: "Rows(color, column=1)", - Sql: `select distinct field, row from λbits where field="color" and column='1' order by row`, - }, - // { // TODO (twg) Not supported by this test, can't reliably get key ordering - // Pql: "Rows(color, previous=green )", - // Sql: `select distinct field, row from λbits where field="color" and row>'green' order by row`, - // }, - { - Pql: "GroupBy(Rows(color))", - Sql: `select f1.field, f1.row,count(*) as cnt from λbits f1 where f1.field='color' group by f1.field,f1.row order by cnt desc`, - }, - { - Pql: "GroupBy(Rows(color),Rows(rating))", - Sql: `select f1.field, f1.row,f2.field, f2.row,count(*) as cnt from λbits f1 inner join λbits f2 on f1.column = f2.column where f1.field='color'and f2.field='rating' group by f1.field,f1.row,f2.field,f2.row order by cnt desc`, - }, - { - Pql: "TopN(color)", - Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, - }, - { - Pql: "TopN(color,n=2)", - Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc limit 2`, - }, - { - Pql: "TopN(color,Row(color=red),n=2)", - Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="color" AND row="red") group by row order by cnt desc limit 2`, - }, - { - Pql: "Topk(color)", - Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, - }, - { - Pql: "TopK(color, k=2, filter=Row(rating=1))", - Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="rating" AND row="1") group by row order by cnt desc limit 2`, - }, - { - Pql: `Min(field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity")`, - }, - { - Pql: `Max(field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select max(val) from λbsi where field="luminosity")`, - }, - { - Pql: `Min(Row(color=orange),field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select distinct column from λbits where field="color" AND row="orange")) and column in (select distinct column from λbits where field="color" AND row="orange")`, - }, - { - Pql: `Min(Intersect(Row(color=red),Row(color=orange)),field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))) and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))`, - }, - { - Pql: `Sum(field="luminosity")`, - Sql: `select sum(val),count(*) from λbsi where field="luminosity"`, - }, - { - Pql: `ConstRow(columns=[1,3,5])`, - Sql: `select column1 as column from (values (1),(3),(5))`, - }, - { - Pql: `UnionRows(Rows(color))`, - Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, - }, - { - Pql: `UnionRows(Rows(color),Rows(rating))`, - Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, - }, - { - Pql: `IncludesColumn(Row(color=red), column=1)`, - Sql: `select count(*)>0 from (select column from(select distinct column from λbits where field="color" AND row="red") where column='1')`, - }, - { - Pql: `Extract(Row(color=red),Rows(color),Rows(rating))`, - Sql: `select sq.field, sq.row, b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row where b.column in (select distinct column from λbits where field="color" AND row="red")`, - }, - { - Pql: `Extract(ConstRow(columns=[1,2,3]))`, - Sql: `select '' as field, '' as row,column from (select column1 as column from (values (1),(2),(3)))`, - }, - } -} -func TestParse(t *testing.T) { - - for _, tst := range GetTests() { - - q, err := pql.NewParser(strings.NewReader(tst.Pql)).Parse() - panicOn(err) - sql, _, _, err := ToSql(q.Calls, "") - panicOn(err) - if sql != tst.Sql { - vv("\npql:%v\ngenerated:\n%v\nexpected:\n%v\n", tst.Pql, sql, tst.Sql) - goon.Dump(q) - } - - } -} diff --git a/cmd/sauron/test_sqlite.sh b/cmd/sauron/test_sqlite.sh deleted file mode 100755 index d0ab85b2f..000000000 --- a/cmd/sauron/test_sqlite.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -db="poc2.db" -sqlite3 ${db} < 0 { - frames := runtime.CallersFrames(pc[:n]) - for i := 0; i <= target; i++ { - contender, more := frames.Next() - if i == target { - f = contender - } - if !more { - break - } - } - } - return f.Function -} diff --git a/go.mod b/go.mod index dcd6b603a..a9a8ef35d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d - github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 + github.com/DataDog/datadog-go v2.2.0+incompatible + github.com/OneOfOne/xxhash v1.2.5 // indirect + github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/beevik/ntp v0.3.0 github.com/benbjohnson/immutable v0.3.0 github.com/cespare/xxhash v1.1.0 @@ -15,26 +17,30 @@ require ( github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 - github.com/gogo/protobuf v1.2.1 + github.com/gogo/protobuf v1.3.1 github.com/golang/protobuf v1.4.2 github.com/google/go-cmp v0.5.2 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect - github.com/gorilla/handlers v1.3.0 - github.com/gorilla/mux v1.7.0 - github.com/hashicorp/memberlist v0.1.3 + github.com/gorilla/handlers v1.4.1 + github.com/gorilla/mux v1.7.3 + github.com/hashicorp/go-immutable-radix v1.1.0 // indirect + github.com/hashicorp/go-msgpack v0.5.5 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/memberlist v0.1.4 github.com/improbable-eng/grpc-web v0.13.0 github.com/kr/text v0.2.0 // indirect github.com/lib/pq v1.8.0 + github.com/miekg/dns v1.1.15 // indirect github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/pelletier/go-toml v1.2.0 + github.com/pelletier/go-toml v1.4.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.1.0 + github.com/prometheus/procfs v0.0.3 // indirect github.com/prometheus/prom2json v1.3.0 github.com/rakyll/statik v0.1.7 - github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/rs/cors v1.7.0 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil/v3 v3.20.11 @@ -54,10 +60,11 @@ require ( golang.org/x/sys v0.0.0-20201214095126-aec9a390925b // indirect golang.org/x/text v0.3.3 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/grpc v1.28.0 + google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 // indirect + google.golang.org/grpc v1.29.1 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.3.0 // indirect - modernc.org/mathutil v1.0.0 + modernc.org/mathutil v1.2.2 modernc.org/strutil v1.0.0 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible ) diff --git a/go.sum b/go.sum index f1ad4ecc5..2b9010aac 100644 --- a/go.sum +++ b/go.sum @@ -16,10 +16,11 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= -github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= -github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/DataDog/datadog-go v2.2.0+incompatible h1:V5BKkxACZLjzHjSgBbr2gvLA2Ae49yhc6CSY7MLy5k4= +github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= +github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -28,8 +29,9 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= @@ -43,6 +45,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -64,10 +68,10 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -86,8 +90,9 @@ github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -97,7 +102,6 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -112,7 +116,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -125,10 +128,10 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= -github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= -github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/handlers v1.4.1 h1:BHvcRGJe/TrL+OqFxoKQGddTgeibiOjaBssV5a/N9sw= +github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -138,23 +141,26 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= +github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc= +github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -174,15 +180,13 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -194,13 +198,14 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= +github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -217,10 +222,12 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -231,6 +238,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -240,27 +248,31 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y= github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= -github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= -github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= @@ -293,17 +305,16 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= @@ -403,7 +414,6 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214095126-aec9a390925b h1:tv7/y4pd+sR8bcNb2D6o7BNU6zjWm0VjQLac+w7fNNM= @@ -417,6 +427,7 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -455,18 +466,19 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU= +google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -485,7 +497,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -498,8 +509,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/http/client_test.go b/http/client_test.go index 87a010501..bb11609d4 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -1462,6 +1462,9 @@ func TestClient_ImportRoaringExists(t *testing.T) { t.Fatal(err) } qr, err := node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + if err != nil { + t.Fatalf(" %v ", err) + } got := qr.Results[0].(*pilosa.Row).Columns() if !reflect.DeepEqual(got, []uint64{}) { t.Fatalf(" Row unexpected columns: got %+v expected: %+v", got, []uint64{}) @@ -1473,6 +1476,9 @@ func TestClient_ImportRoaringExists(t *testing.T) { expected := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537} qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + if err != nil { + t.Fatalf("Query error: %+v", err) + } got = qr.Results[0].(*pilosa.Row).Columns() if !reflect.DeepEqual(got, expected) { t.Fatalf("All unexpected columns: got %+v expected: %+v", got, expected) diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 4773ef666..5794a05d0 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -16,9 +16,10 @@ package pilosa import ( "bytes" - "github.com/pilosa/pilosa/v2/roaring" "reflect" "testing" + + "github.com/pilosa/pilosa/v2/roaring" ) func TestValidateName(t *testing.T) { @@ -73,22 +74,32 @@ func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, e func TestAPI_CombineForExistence(t *testing.T) { bm := roaring.NewBitmap() - bm.Add(pos(1, 1)) - bm.Add(pos(1, 2)) - bm.Add(pos(1, 65537)) //make sure to cross container boundary - bm.Add(pos(1, 65538)) - bm.Add(pos(2, 1)) - bm.Add(pos(2, 2)) - bm.Add(pos(2, 65537)) - bm.Add(pos(2, 65538)) + _, err := bm.Add(pos(1, 1)) + panicOn(err) + _, err = bm.Add(pos(1, 2)) + panicOn(err) + _, err = bm.Add(pos(1, 65537)) //make sure to cross container boundary + panicOn(err) + _, err = bm.Add(pos(1, 65538)) + panicOn(err) + _, err = bm.Add(pos(2, 1)) + panicOn(err) + _, err = bm.Add(pos(2, 2)) + panicOn(err) + _, err = bm.Add(pos(2, 65537)) + panicOn(err) + _, err = bm.Add(pos(2, 65538)) + panicOn(err) buf := new(bytes.Buffer) - _, err := bm.WriteTo(buf) + _, err = bm.WriteTo(buf) panicOn(err) raw := buf.Bytes() - results := combineForExistence(raw) + results, err := combineForExistence(raw) + panicOn(err) bm2 := roaring.NewBitmap() - bm2.ImportRoaringBits(results, false, false, 1< Date: Mon, 22 Feb 2021 08:33:52 -0600 Subject: [PATCH 04/28] update go mod to fix cors bug --- go.mod | 25 +++++++---------- go.sum | 85 +++++++++++++++++++++++++--------------------------------- 2 files changed, 46 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index a9a8ef35d..dcd6b603a 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,7 @@ replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d - github.com/DataDog/datadog-go v2.2.0+incompatible - github.com/OneOfOne/xxhash v1.2.5 // indirect - github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect + github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 github.com/beevik/ntp v0.3.0 github.com/benbjohnson/immutable v0.3.0 github.com/cespare/xxhash v1.1.0 @@ -17,30 +15,26 @@ require ( github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 - github.com/gogo/protobuf v1.3.1 + github.com/gogo/protobuf v1.2.1 github.com/golang/protobuf v1.4.2 github.com/google/go-cmp v0.5.2 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect - github.com/gorilla/handlers v1.4.1 - github.com/gorilla/mux v1.7.3 - github.com/hashicorp/go-immutable-radix v1.1.0 // indirect - github.com/hashicorp/go-msgpack v0.5.5 // indirect - github.com/hashicorp/go-sockaddr v1.0.2 // indirect - github.com/hashicorp/memberlist v0.1.4 + github.com/gorilla/handlers v1.3.0 + github.com/gorilla/mux v1.7.0 + github.com/hashicorp/memberlist v0.1.3 github.com/improbable-eng/grpc-web v0.13.0 github.com/kr/text v0.2.0 // indirect github.com/lib/pq v1.8.0 - github.com/miekg/dns v1.1.15 // indirect github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/pelletier/go-toml v1.4.0 + github.com/pelletier/go-toml v1.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.1.0 - github.com/prometheus/procfs v0.0.3 // indirect github.com/prometheus/prom2json v1.3.0 github.com/rakyll/statik v0.1.7 + github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/rs/cors v1.7.0 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil/v3 v3.20.11 @@ -60,11 +54,10 @@ require ( golang.org/x/sys v0.0.0-20201214095126-aec9a390925b // indirect golang.org/x/text v0.3.3 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 // indirect - google.golang.org/grpc v1.29.1 + google.golang.org/grpc v1.28.0 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.3.0 // indirect - modernc.org/mathutil v1.2.2 + modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible ) diff --git a/go.sum b/go.sum index 2b9010aac..f1ad4ecc5 100644 --- a/go.sum +++ b/go.sum @@ -16,11 +16,10 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= -github.com/DataDog/datadog-go v2.2.0+incompatible h1:V5BKkxACZLjzHjSgBbr2gvLA2Ae49yhc6CSY7MLy5k4= -github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= +github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= -github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -29,9 +28,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= @@ -45,8 +43,6 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -68,10 +64,10 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -90,9 +86,8 @@ github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -102,6 +97,7 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -116,6 +112,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -128,10 +125,10 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/handlers v1.4.1 h1:BHvcRGJe/TrL+OqFxoKQGddTgeibiOjaBssV5a/N9sw= -github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= +github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= +github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -141,26 +138,23 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc= -github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= -github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= -github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -180,13 +174,15 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -198,14 +194,13 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= -github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -222,12 +217,10 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= -github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -238,7 +231,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -248,31 +240,27 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y= github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= @@ -305,16 +293,17 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= @@ -414,6 +403,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214095126-aec9a390925b h1:tv7/y4pd+sR8bcNb2D6o7BNU6zjWm0VjQLac+w7fNNM= @@ -427,7 +417,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -466,19 +455,18 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU= -google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -497,6 +485,7 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -509,8 +498,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY= -modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= From 155a4b4a316ccb0baf8d15321b668286dc7d66bf Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 22 Feb 2021 10:07:32 -0600 Subject: [PATCH 05/28] comment clarification --- api.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index f67ecf100..34976f0df 100644 --- a/api.go +++ b/api.go @@ -423,9 +423,10 @@ func importWorker(importWork chan importJob) { case RequestActionSet: fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) data := viewData - if fileMagic != roaring.MagicNumber { // if pilosa roaring format - // must make a copy of data to operate on locally on standard roaring format. - // field.importRoaring changes the standard roaring run format to pilosa roaring + if fileMagic != roaring.MagicNumber { + // if the view data arrives is in the "standard" roaring format, we must + // make a copy of data in order allow for the convertion to the pilosa roaring run format + // in field.importRoaring data = make([]byte, len(viewData)) copy(data, viewData) } From 1b87b7148ee7e8937cd3e71e3b97a2fe44cdc3ed Mon Sep 17 00:00:00 2001 From: tgruben Date: Mon, 22 Feb 2021 11:06:55 -0600 Subject: [PATCH 06/28] Update api.go Co-authored-by: Matthew Jaffee --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 34976f0df..7c3bc8ceb 100644 --- a/api.go +++ b/api.go @@ -425,7 +425,7 @@ func importWorker(importWork chan importJob) { data := viewData if fileMagic != roaring.MagicNumber { // if the view data arrives is in the "standard" roaring format, we must - // make a copy of data in order allow for the convertion to the pilosa roaring run format + // make a copy of data in order allow for the conversion to the pilosa roaring run format // in field.importRoaring data = make([]byte, len(viewData)) copy(data, viewData) From c2da1d267158171338acc74a47a48b0b9195429c Mon Sep 17 00:00:00 2001 From: tgruben Date: Mon, 22 Feb 2021 11:07:41 -0600 Subject: [PATCH 07/28] Update api.go Co-authored-by: Matthew Jaffee --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 7c3bc8ceb..3a7dc36cf 100644 --- a/api.go +++ b/api.go @@ -465,7 +465,7 @@ func importWorker(importWork chan importJob) { } } -// merge all rows to singled existence row +// combineForExistence unions all rows in the fragment to be imported into a single row to update the existence field. TODO: It would probably be more efficient to only unmarshal the input data once, and use the calculated existence Bitmap directly rather than returning it to bytes, but most of our ingest paths update existence separately, so it's more important that this just be obviously correct at the moment. func combineForExistence(inputRoaringData []byte) ([]byte, error) { rowSize := uint64(1 << shardVsContainerExponent) rit, err := roaring.NewRoaringIterator(inputRoaringData) From 5cbbb7996e28b09054fa55426c4b5fc582701405 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 22 Feb 2021 11:37:30 -0600 Subject: [PATCH 08/28] jaffee test handling suggestions --- pilosa_internal_test.go | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 5794a05d0..693dc52cb 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -73,30 +73,15 @@ func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, e } func TestAPI_CombineForExistence(t *testing.T) { - bm := roaring.NewBitmap() - _, err := bm.Add(pos(1, 1)) - panicOn(err) - _, err = bm.Add(pos(1, 2)) - panicOn(err) - _, err = bm.Add(pos(1, 65537)) //make sure to cross container boundary - panicOn(err) - _, err = bm.Add(pos(1, 65538)) - panicOn(err) - _, err = bm.Add(pos(2, 1)) - panicOn(err) - _, err = bm.Add(pos(2, 2)) - panicOn(err) - _, err = bm.Add(pos(2, 65537)) - panicOn(err) - _, err = bm.Add(pos(2, 65538)) - panicOn(err) - + bm := roaring.NewBitmap(pos(1, 1), pos(1, 2), pos(1, 65537), pos(1, 65538), pos(2, 1), pos(2, 2), pos(2, 65537), pos(2, 65538)) buf := new(bytes.Buffer) - _, err = bm.WriteTo(buf) + _, err := bm.WriteTo(buf) panicOn(err) raw := buf.Bytes() results, err := combineForExistence(raw) - panicOn(err) + if err != nil { + t.Fatalf("failure to combine: %v", err) + } bm2 := roaring.NewBitmap() _, _, err = bm2.ImportRoaringBits(results, false, false, 1< Date: Mon, 22 Feb 2021 11:42:48 -0600 Subject: [PATCH 09/28] Update roaring/roaring.go Co-authored-by: Matthew Jaffee --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 37adfc2d5..39bdbc714 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2282,7 +2282,7 @@ func (b *Bitmap) ImportRoaringBits(data []byte, clear bool, log bool, rowSize ui func (b *Bitmap) MergeRoaringRawIteratorIntoExists(itr RoaringIterator, rowSize uint64) error { if itr == nil { - return errors.New("bad roaring iterator, but don't know why") + return errors.New("nil RoaringIterator passed to MergeRoaringRawIteratorIntoExists") } var synthC Container importUpdater := func(oldC *Container, existed bool) (newC *Container, write bool) { From d220426686d51a473b0b999c56737b9f46e5e557 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 22 Feb 2021 11:50:01 -0600 Subject: [PATCH 10/28] doc comment --- roaring/roaring.go | 80 ++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 39bdbc714..1ee979bde 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2279,46 +2279,50 @@ func (b *Bitmap) ImportRoaringBits(data []byte, clear bool, log bool, rowSize ui } return b.ImportRoaringRawIterator(itr, clear, log, rowSize) } + +// MergeRoaringRawIteratorIntoExists is a special merge iterator that flattens the results onto a +// single row which is used for determining existence. All row references are removed and only the column +// is considered. func (b *Bitmap) MergeRoaringRawIteratorIntoExists(itr RoaringIterator, rowSize uint64) error { - if itr == nil { - return errors.New("nil RoaringIterator passed to MergeRoaringRawIteratorIntoExists") - } - var synthC Container - importUpdater := func(oldC *Container, existed bool) (newC *Container, write bool) { - existN := oldC.N() - if existN == MaxContainerVal+1 { - return oldC, false - } - if existN == 0 { - newerC := synthC.Clone() - return newerC, true - } - newC = oldC.unionInPlace(&synthC) - if newC.typeID == ContainerBitmap { - newC.Repair() - } - if newC.N() != existN { - return newC, true - } - return oldC, false - } - itrKey, itrCType, itrN, itrLen, itrPointer, itrErr := itr.Next() - for itrErr == nil { - synthC.typeID = itrCType - synthC.n = int32(itrN) - synthC.len = int32(itrLen) - synthC.cap = int32(itrLen) - synthC.pointer = itrPointer - b.Containers.Update(itrKey%rowSize, importUpdater) - itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next() - } - // note: if we get a non-EOF err, it's possible that we made SOME - // changes but didn't log them. I don't have a good solution to this. - if itrErr != io.EOF { - return itrErr - } - return nil + if itr == nil { + return errors.New("nil RoaringIterator passed to MergeRoaringRawIteratorIntoExists") + } + var synthC Container + importUpdater := func(oldC *Container, existed bool) (newC *Container, write bool) { + existN := oldC.N() + if existN == MaxContainerVal+1 { + return oldC, false + } + if existN == 0 { + newerC := synthC.Clone() + return newerC, true + } + newC = oldC.unionInPlace(&synthC) + if newC.typeID == ContainerBitmap { + newC.Repair() + } + if newC.N() != existN { + return newC, true + } + return oldC, false + } + itrKey, itrCType, itrN, itrLen, itrPointer, itrErr := itr.Next() + for itrErr == nil { + synthC.typeID = itrCType + synthC.n = int32(itrN) + synthC.len = int32(itrLen) + synthC.cap = int32(itrLen) + synthC.pointer = itrPointer + b.Containers.Update(itrKey%rowSize, importUpdater) + itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next() + } + // note: if we get a non-EOF err, it's possible that we made SOME + // changes but didn't log them. I don't have a good solution to this. + if itrErr != io.EOF { + return itrErr + } + return nil } func (b *Bitmap) ImportRoaringRawIterator(itr RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { From 081d86c04b1e128789c7ea0c06fca765f54b4e85 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 22 Feb 2021 15:06:45 -0600 Subject: [PATCH 11/28] mixed row test --- pilosa_internal_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 693dc52cb..b984d5680 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -73,7 +73,7 @@ func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, e } func TestAPI_CombineForExistence(t *testing.T) { - bm := roaring.NewBitmap(pos(1, 1), pos(1, 2), pos(1, 65537), pos(1, 65538), pos(2, 1), pos(2, 2), pos(2, 65537), pos(2, 65538)) + bm := roaring.NewBitmap(pos(1, 1), pos(1, 2), pos(1, 3), pos(1, 65537), pos(1, 65538), pos(2, 1), pos(2, 2), pos(2, 5), pos(2, 65537), pos(2, 65538)) buf := new(bytes.Buffer) _, err := bm.WriteTo(buf) panicOn(err) @@ -85,7 +85,7 @@ func TestAPI_CombineForExistence(t *testing.T) { bm2 := roaring.NewBitmap() _, _, err = bm2.ImportRoaringBits(results, false, false, 1< Date: Mon, 22 Feb 2021 16:16:11 -0600 Subject: [PATCH 12/28] Add ignoreLimit argument to executeGroupByShard --- executor.go | 8 +++++--- executor_test.go | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index 34fc5c87a..bc65a978c 100644 --- a/executor.go +++ b/executor.go @@ -2876,10 +2876,12 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c } } + ignoreLimit := sorter != nil // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (_ interface{}, err error) { - return e.executeGroupByShard(ctx, qcx, index, c, filter, shard, childRows, bases) + return e.executeGroupByShard(ctx, qcx, index, c, filter, shard, childRows, bases, ignoreLimit) } + // Merge returned results at coordinating node. reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other := findGroupCounts(prev) @@ -3434,7 +3436,7 @@ func applyConditionToGroupCounts(gcs []GroupCount, subj string, cond *pql.Condit return gcs[:i] } -func (e *executor) executeGroupByShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs, bases map[int]int64) (_ []GroupCount, err error) { +func (e *executor) executeGroupByShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs, bases map[int]int64, ignoreLimit bool) (_ []GroupCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupByShard") defer span.Finish() @@ -3464,7 +3466,7 @@ func (e *executor) executeGroupByShard(ctx context.Context, qcx *Qcx, index stri limit := int(^uint(0) >> 1) if lim, hasLimit, err := c.UintArg("limit"); err != nil { return nil, err - } else if hasLimit { + } else if !ignoreLimit && hasLimit { limit = int(lim) } diff --git a/executor_test.go b/executor_test.go index 3cb51cac7..d3b959302 100644 --- a/executor_test.go +++ b/executor_test.go @@ -7013,6 +7013,17 @@ func variousQueries(t *testing.T, clusterSize int) { {"icecream", "userF"}, }) + // Create and populate "dinner" field. + c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "dinner", pilosa.OptFieldKeys()) + c.ImportKeyKey(t, "users", "dinner", [][2]string{ + {"leftovers", "userB"}, + {"pizza", "userA"}, + {"pizza", "userB"}, + {"chinese", "userA"}, + {"chinese", "userB"}, + {"chinese", "userF"}, + }) + // Create and populate "places_visited" time field. c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM"))) ts2019Jan01 := int64(1546300800) * 1e+9 // 2019 January 1st 0:00:00 @@ -7388,6 +7399,12 @@ pangolin,1,100 0,1,1 5,1,1 10,1,1 +`, + }, + { + query: "GroupBy(Rows(field=dinner), sort=\"count desc\", limit=2)", + csvVerifier: `chinese,3 +pizza,2 `, }, } From 5e477e107acb5c2af0cceaf9c5b02c62f9dd506b Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 23 Feb 2021 08:32:07 -0700 Subject: [PATCH 13/28] Remove unused RBFTx.frag field --- rbf.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/rbf.go b/rbf.go index 4fa2f8900..3ae1246d7 100644 --- a/rbf.go +++ b/rbf.go @@ -184,7 +184,6 @@ type RBFTx struct { // initialIndex is only a debugging aid. Transactions // can cross indexes. It can be left empty without consequence. initialIndex string - frag *fragment tx *rbf.Tx o Txo sn int64 // serial number @@ -520,7 +519,6 @@ func (w *RbfDBWrapper) NewTx(write bool, initialIndex string, o Txo) (_ Tx, err rtx := &RBFTx{ tx: tx, initialIndex: initialIndex, - frag: o.Fragment, o: o, sn: sn, Db: w, From db01237904a72e6d66668936f66a970b74f4d1a2 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Wed, 24 Feb 2021 15:29:10 -0600 Subject: [PATCH 14/28] allow 'field=' for TopN() --- lattice | 2 +- pql/pql.peg | 2 +- pql/pql.peg.go | 546 ++++++++++++++++++++++++--------------------- pql/pqlpeg_test.go | 13 ++ 4 files changed, 307 insertions(+), 256 deletions(-) diff --git a/lattice b/lattice index fa773628a..28c2313ec 160000 --- a/lattice +++ b/lattice @@ -1 +1 @@ -Subproject commit fa773628a276e2590785a87fbc236c7e88ea6284 +Subproject commit 28c2313ecfcd7e083d42d4e409483e968b4c421b diff --git a/pql/pql.peg b/pql/pql.peg index ec6241398..8d7841f5f 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -54,7 +54,7 @@ singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )* fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )* field <- { p.addField(text) } reserved <- '_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field' -posfield <- { p.addPosStr("_field", text) } +posfield <- 'field='? { p.addPosStr("_field", text) } col <- < digits > {p.addPosNum("_col", text)} / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", text)} / < '"' doublequotedstring '"' > {p.addPosStr("_col", text)} diff --git a/pql/pql.peg.go b/pql/pql.peg.go index e761e9e60..ed9a8871e 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -8,6 +8,7 @@ import ( "os" "sort" "strconv" + "strings" ) const endSymbol rune = 1114112 @@ -239,7 +240,7 @@ func (node *node32) print(w io.Writer, pretty bool, buffer string) { if !pretty { fmt.Fprintf(w, "%v %v\n", rule, quote) } else { - fmt.Fprintf(w, "\x1B[34m%v\x1B[m %v\n", rule, quote) + fmt.Fprintf(w, "\x1B[36m%v\x1B[m %v\n", rule, quote) } if node.up != nil { print(node.up, depth+1) @@ -414,6 +415,12 @@ func (p *PQL) WriteSyntaxTree(w io.Writer) { p.tokens32.WriteSyntaxTree(w, p.Buffer) } +func (p *PQL) SprintSyntaxTree() string { + var bldr strings.Builder + p.WriteSyntaxTree(&bldr) + return bldr.String() +} + func (p *PQL) Execute() { buffer, _buffer, text, begin, end := p.Buffer, p.buffer, "", 0, 0 for _, token := range p.Tokens() { @@ -3277,17 +3284,48 @@ func (p *PQL) Init(options ...func(*PQL) error) error { }, /* 17 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 18 posfield <- <( Action51)> */ + /* 18 posfield <- <(('f' 'i' 'e' 'l' 'd' '=')? Action51)> */ func() bool { position363, tokenIndex363 := position, tokenIndex { position364 := position { - position365 := position + position365, tokenIndex365 := position, tokenIndex + if buffer[position] != rune('f') { + goto l365 + } + position++ + if buffer[position] != rune('i') { + goto l365 + } + position++ + if buffer[position] != rune('e') { + goto l365 + } + position++ + if buffer[position] != rune('l') { + goto l365 + } + position++ + if buffer[position] != rune('d') { + goto l365 + } + position++ + if buffer[position] != rune('=') { + goto l365 + } + position++ + goto l366 + l365: + position, tokenIndex = position365, tokenIndex365 + } + l366: + { + position367 := position if !_rules[rulefieldExpr]() { goto l363 } - add(rulePegText, position365) + add(rulePegText, position367) } { add(ruleAction51, position) @@ -3301,175 +3339,153 @@ func (p *PQL) Init(options ...func(*PQL) error) error { }, /* 19 col <- <(( Action52) / (<('\'' singlequotedstring '\'')> Action53) / (<('"' doublequotedstring '"')> Action54))> */ func() bool { - position367, tokenIndex367 := position, tokenIndex + position369, tokenIndex369 := position, tokenIndex { - position368 := position + position370 := position { - position369, tokenIndex369 := position, tokenIndex + position371, tokenIndex371 := position, tokenIndex { - position371 := position + position373 := position if !_rules[ruledigits]() { - goto l370 + goto l372 } - add(rulePegText, position371) + add(rulePegText, position373) } { add(ruleAction52, position) } - goto l369 - l370: - position, tokenIndex = position369, tokenIndex369 + goto l371 + l372: + position, tokenIndex = position371, tokenIndex371 { - position374 := position + position376 := position if buffer[position] != rune('\'') { - goto l373 + goto l375 } position++ if !_rules[rulesinglequotedstring]() { - goto l373 + goto l375 } if buffer[position] != rune('\'') { - goto l373 - } - position++ - add(rulePegText, position374) - } - { - add(ruleAction53, position) - } - goto l369 - l373: - position, tokenIndex = position369, tokenIndex369 - { - position376 := position - if buffer[position] != rune('"') { - goto l367 - } - position++ - if !_rules[ruledoublequotedstring]() { - goto l367 - } - if buffer[position] != rune('"') { - goto l367 + goto l375 } position++ add(rulePegText, position376) } + { + add(ruleAction53, position) + } + goto l371 + l375: + position, tokenIndex = position371, tokenIndex371 + { + position378 := position + if buffer[position] != rune('"') { + goto l369 + } + position++ + if !_rules[ruledoublequotedstring]() { + goto l369 + } + if buffer[position] != rune('"') { + goto l369 + } + position++ + add(rulePegText, position378) + } { add(ruleAction54, position) } } - l369: - add(rulecol, position368) + l371: + add(rulecol, position370) } return true - l367: - position, tokenIndex = position367, tokenIndex367 + l369: + position, tokenIndex = position369, tokenIndex369 return false }, /* 20 row <- <(( Action55) / (<('\'' singlequotedstring '\'')> Action56) / (<('"' doublequotedstring '"')> Action57))> */ nil, /* 21 open <- <('(' sp)> */ - func() bool { - position379, tokenIndex379 := position, tokenIndex - { - position380 := position - if buffer[position] != rune('(') { - goto l379 - } - position++ - if !_rules[rulesp]() { - goto l379 - } - add(ruleopen, position380) - } - return true - l379: - position, tokenIndex = position379, tokenIndex379 - return false - }, - /* 22 close <- <(sp ')' sp)> */ func() bool { position381, tokenIndex381 := position, tokenIndex { position382 := position - if !_rules[rulesp]() { - goto l381 - } - if buffer[position] != rune(')') { + if buffer[position] != rune('(') { goto l381 } position++ if !_rules[rulesp]() { goto l381 } - add(ruleclose, position382) + add(ruleopen, position382) } return true l381: position, tokenIndex = position381, tokenIndex381 return false }, + /* 22 close <- <(sp ')' sp)> */ + func() bool { + position383, tokenIndex383 := position, tokenIndex + { + position384 := position + if !_rules[rulesp]() { + goto l383 + } + if buffer[position] != rune(')') { + goto l383 + } + position++ + if !_rules[rulesp]() { + goto l383 + } + add(ruleclose, position384) + } + return true + l383: + position, tokenIndex = position383, tokenIndex383 + return false + }, /* 23 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position384 := position - l385: + position386 := position + l387: { - position386, tokenIndex386 := position, tokenIndex + position388, tokenIndex388 := position, tokenIndex { - position387, tokenIndex387 := position, tokenIndex + position389, tokenIndex389 := position, tokenIndex if buffer[position] != rune(' ') { + goto l390 + } + position++ + goto l389 + l390: + position, tokenIndex = position389, tokenIndex389 + if buffer[position] != rune('\t') { + goto l391 + } + position++ + goto l389 + l391: + position, tokenIndex = position389, tokenIndex389 + if buffer[position] != rune('\n') { goto l388 } position++ - goto l387 - l388: - position, tokenIndex = position387, tokenIndex387 - if buffer[position] != rune('\t') { - goto l389 - } - position++ - goto l387 - l389: - position, tokenIndex = position387, tokenIndex387 - if buffer[position] != rune('\n') { - goto l386 - } - position++ } - l387: - goto l385 - l386: - position, tokenIndex = position386, tokenIndex386 + l389: + goto l387 + l388: + position, tokenIndex = position388, tokenIndex388 } - add(rulesp, position384) + add(rulesp, position386) } return true }, /* 24 eq <- <(sp '=' sp)> */ - func() bool { - position390, tokenIndex390 := position, tokenIndex - { - position391 := position - if !_rules[rulesp]() { - goto l390 - } - if buffer[position] != rune('=') { - goto l390 - } - position++ - if !_rules[rulesp]() { - goto l390 - } - add(ruleeq, position391) - } - return true - l390: - position, tokenIndex = position390, tokenIndex390 - return false - }, - /* 25 comma <- <(sp ',' sp)> */ func() bool { position392, tokenIndex392 := position, tokenIndex { @@ -3477,298 +3493,302 @@ func (p *PQL) Init(options ...func(*PQL) error) error { if !_rules[rulesp]() { goto l392 } - if buffer[position] != rune(',') { + if buffer[position] != rune('=') { goto l392 } position++ if !_rules[rulesp]() { goto l392 } - add(rulecomma, position393) + add(ruleeq, position393) } return true l392: position, tokenIndex = position392, tokenIndex392 return false }, + /* 25 comma <- <(sp ',' sp)> */ + func() bool { + position394, tokenIndex394 := position, tokenIndex + { + position395 := position + if !_rules[rulesp]() { + goto l394 + } + if buffer[position] != rune(',') { + goto l394 + } + position++ + if !_rules[rulesp]() { + goto l394 + } + add(rulecomma, position395) + } + return true + l394: + position, tokenIndex = position394, tokenIndex394 + return false + }, /* 26 lbrack <- <('[' sp)> */ nil, /* 27 rbrack <- <(sp ']' sp)> */ nil, /* 28 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { - position396, tokenIndex396 := position, tokenIndex + position398, tokenIndex398 := position, tokenIndex { - position397 := position + position399 := position { - position398, tokenIndex398 := position, tokenIndex + position400, tokenIndex400 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l399 + goto l401 } position++ - goto l398 - l399: - position, tokenIndex = position398, tokenIndex398 + goto l400 + l401: + position, tokenIndex = position400, tokenIndex400 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l396 + goto l398 } position++ } - l398: l400: + l402: { - position401, tokenIndex401 := position, tokenIndex + position403, tokenIndex403 := position, tokenIndex { - position402, tokenIndex402 := position, tokenIndex + position404, tokenIndex404 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l405 + } + position++ + goto l404 + l405: + position, tokenIndex = position404, tokenIndex404 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l406 + } + position++ + goto l404 + l406: + position, tokenIndex = position404, tokenIndex404 + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l403 } position++ - goto l402 - l403: - position, tokenIndex = position402, tokenIndex402 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l404 - } - position++ - goto l402 - l404: - position, tokenIndex = position402, tokenIndex402 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l401 - } - position++ } - l402: - goto l400 - l401: - position, tokenIndex = position401, tokenIndex401 + l404: + goto l402 + l403: + position, tokenIndex = position403, tokenIndex403 } - add(ruleIDENT, position397) + add(ruleIDENT, position399) } return true - l396: - position, tokenIndex = position396, tokenIndex396 + l398: + position, tokenIndex = position398, tokenIndex398 return false }, /* 29 digits <- <[0-9]+> */ func() bool { - position405, tokenIndex405 := position, tokenIndex + position407, tokenIndex407 := position, tokenIndex { - position406 := position + position408 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l405 + goto l407 } position++ - l407: + l409: { - position408, tokenIndex408 := position, tokenIndex + position410, tokenIndex410 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l408 + goto l410 } position++ - goto l407 - l408: - position, tokenIndex = position408, tokenIndex408 + goto l409 + l410: + position, tokenIndex = position410, tokenIndex410 } - add(ruledigits, position406) + add(ruledigits, position408) } return true - l405: - position, tokenIndex = position405, tokenIndex405 + l407: + position, tokenIndex = position407, tokenIndex407 return false }, /* 30 signedDigits <- <('-'? digits)> */ nil, /* 31 decimal <- <((signedDigits ('.' digits?)?) / ('-'? '.' digits))> */ func() bool { - position410, tokenIndex410 := position, tokenIndex + position412, tokenIndex412 := position, tokenIndex { - position411 := position + position413 := position { - position412, tokenIndex412 := position, tokenIndex + position414, tokenIndex414 := position, tokenIndex { - position414 := position + position416 := position { - position415, tokenIndex415 := position, tokenIndex + position417, tokenIndex417 := position, tokenIndex if buffer[position] != rune('-') { - goto l415 + goto l417 } position++ - goto l416 - l415: - position, tokenIndex = position415, tokenIndex415 + goto l418 + l417: + position, tokenIndex = position417, tokenIndex417 } - l416: + l418: if !_rules[ruledigits]() { - goto l413 + goto l415 } - add(rulesignedDigits, position414) + add(rulesignedDigits, position416) } { - position417, tokenIndex417 := position, tokenIndex + position419, tokenIndex419 := position, tokenIndex if buffer[position] != rune('.') { - goto l417 + goto l419 } position++ { - position419, tokenIndex419 := position, tokenIndex + position421, tokenIndex421 := position, tokenIndex if !_rules[ruledigits]() { - goto l419 + goto l421 } - goto l420 - l419: - position, tokenIndex = position419, tokenIndex419 + goto l422 + l421: + position, tokenIndex = position421, tokenIndex421 } - l420: - goto l418 - l417: - position, tokenIndex = position417, tokenIndex417 + l422: + goto l420 + l419: + position, tokenIndex = position419, tokenIndex419 } - l418: - goto l412 - l413: - position, tokenIndex = position412, tokenIndex412 + l420: + goto l414 + l415: + position, tokenIndex = position414, tokenIndex414 { - position421, tokenIndex421 := position, tokenIndex + position423, tokenIndex423 := position, tokenIndex if buffer[position] != rune('-') { - goto l421 + goto l423 } position++ - goto l422 - l421: - position, tokenIndex = position421, tokenIndex421 + goto l424 + l423: + position, tokenIndex = position423, tokenIndex423 } - l422: + l424: if buffer[position] != rune('.') { - goto l410 + goto l412 } position++ if !_rules[ruledigits]() { - goto l410 + goto l412 } } - l412: - add(ruledecimal, position411) + l414: + add(ruledecimal, position413) } return true - l410: - position, tokenIndex = position410, tokenIndex410 + l412: + position, tokenIndex = position412, tokenIndex412 return false }, /* 32 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position423, tokenIndex423 := position, tokenIndex + position425, tokenIndex425 := position, tokenIndex { - position424 := position + position426 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if buffer[position] != rune('-') { - goto l423 + goto l425 } position++ { - position425, tokenIndex425 := position, tokenIndex + position427, tokenIndex427 := position, tokenIndex if buffer[position] != rune('0') { - goto l426 + goto l428 } position++ - goto l425 - l426: - position, tokenIndex = position425, tokenIndex425 + goto l427 + l428: + position, tokenIndex = position427, tokenIndex427 if buffer[position] != rune('1') { - goto l423 + goto l425 } position++ } - l425: + l427: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if buffer[position] != rune('-') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if buffer[position] != rune('T') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if buffer[position] != rune(':') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ - add(ruletimestampbasicfmt, position424) + add(ruletimestampbasicfmt, position426) } return true - l423: - position, tokenIndex = position423, tokenIndex423 + l425: + position, tokenIndex = position425, tokenIndex425 return false }, /* 33 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position427, tokenIndex427 := position, tokenIndex + position429, tokenIndex429 := position, tokenIndex { - position428 := position + position430 := position { - position429, tokenIndex429 := position, tokenIndex + position431, tokenIndex431 := position, tokenIndex if buffer[position] != rune('"') { - goto l430 - } - position++ - { - position431 := position - if !_rules[ruletimestampbasicfmt]() { - goto l430 - } - add(rulePegText, position431) - } - if buffer[position] != rune('"') { - goto l430 - } - position++ - goto l429 - l430: - position, tokenIndex = position429, tokenIndex429 - if buffer[position] != rune('\'') { goto l432 } position++ @@ -3779,27 +3799,45 @@ func (p *PQL) Init(options ...func(*PQL) error) error { } add(rulePegText, position433) } - if buffer[position] != rune('\'') { + if buffer[position] != rune('"') { goto l432 } position++ - goto l429 + goto l431 l432: - position, tokenIndex = position429, tokenIndex429 + position, tokenIndex = position431, tokenIndex431 + if buffer[position] != rune('\'') { + goto l434 + } + position++ { - position434 := position + position435 := position if !_rules[ruletimestampbasicfmt]() { - goto l427 + goto l434 } - add(rulePegText, position434) + add(rulePegText, position435) + } + if buffer[position] != rune('\'') { + goto l434 + } + position++ + goto l431 + l434: + position, tokenIndex = position431, tokenIndex431 + { + position436 := position + if !_rules[ruletimestampbasicfmt]() { + goto l429 + } + add(rulePegText, position436) } } - l429: - add(ruletimestampfmt, position428) + l431: + add(ruletimestampfmt, position430) } return true - l427: - position, tokenIndex = position427, tokenIndex427 + l429: + position, tokenIndex = position429, tokenIndex429 return false }, /* 34 timestamp <- <( Action58)> */ diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 12b9e4b15..1927a44fe 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -522,6 +522,19 @@ func TestPQLDeepEquality(t *testing.T) { {Name: "Row"}, }, }}, + { + name: "TopNwithField=", + call: "TopN(field=myfield, Row(), a=7)", + exp: &Call{ + Name: "TopN", + Args: map[string]interface{}{ + "a": int64(7), + "_field": "myfield", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, { name: "RangeEQ", call: "Row(a==7)", From 6b75a8b50099f56f36942626187438141b7fc5b6 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Thu, 25 Feb 2021 11:01:22 -0600 Subject: [PATCH 15/28] Revert "allow 'field=' for TopN()" This reverts commit db01237904a72e6d66668936f66a970b74f4d1a2. --- lattice | 2 +- pql/pql.peg | 2 +- pql/pql.peg.go | 546 +++++++++++++++++++++------------------------ pql/pqlpeg_test.go | 13 -- 4 files changed, 256 insertions(+), 307 deletions(-) diff --git a/lattice b/lattice index 28c2313ec..fa773628a 160000 --- a/lattice +++ b/lattice @@ -1 +1 @@ -Subproject commit 28c2313ecfcd7e083d42d4e409483e968b4c421b +Subproject commit fa773628a276e2590785a87fbc236c7e88ea6284 diff --git a/pql/pql.peg b/pql/pql.peg index 8d7841f5f..ec6241398 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -54,7 +54,7 @@ singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )* fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )* field <- { p.addField(text) } reserved <- '_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field' -posfield <- 'field='? { p.addPosStr("_field", text) } +posfield <- { p.addPosStr("_field", text) } col <- < digits > {p.addPosNum("_col", text)} / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", text)} / < '"' doublequotedstring '"' > {p.addPosStr("_col", text)} diff --git a/pql/pql.peg.go b/pql/pql.peg.go index ed9a8871e..e761e9e60 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -8,7 +8,6 @@ import ( "os" "sort" "strconv" - "strings" ) const endSymbol rune = 1114112 @@ -240,7 +239,7 @@ func (node *node32) print(w io.Writer, pretty bool, buffer string) { if !pretty { fmt.Fprintf(w, "%v %v\n", rule, quote) } else { - fmt.Fprintf(w, "\x1B[36m%v\x1B[m %v\n", rule, quote) + fmt.Fprintf(w, "\x1B[34m%v\x1B[m %v\n", rule, quote) } if node.up != nil { print(node.up, depth+1) @@ -415,12 +414,6 @@ func (p *PQL) WriteSyntaxTree(w io.Writer) { p.tokens32.WriteSyntaxTree(w, p.Buffer) } -func (p *PQL) SprintSyntaxTree() string { - var bldr strings.Builder - p.WriteSyntaxTree(&bldr) - return bldr.String() -} - func (p *PQL) Execute() { buffer, _buffer, text, begin, end := p.Buffer, p.buffer, "", 0, 0 for _, token := range p.Tokens() { @@ -3284,48 +3277,17 @@ func (p *PQL) Init(options ...func(*PQL) error) error { }, /* 17 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 18 posfield <- <(('f' 'i' 'e' 'l' 'd' '=')? Action51)> */ + /* 18 posfield <- <( Action51)> */ func() bool { position363, tokenIndex363 := position, tokenIndex { position364 := position { - position365, tokenIndex365 := position, tokenIndex - if buffer[position] != rune('f') { - goto l365 - } - position++ - if buffer[position] != rune('i') { - goto l365 - } - position++ - if buffer[position] != rune('e') { - goto l365 - } - position++ - if buffer[position] != rune('l') { - goto l365 - } - position++ - if buffer[position] != rune('d') { - goto l365 - } - position++ - if buffer[position] != rune('=') { - goto l365 - } - position++ - goto l366 - l365: - position, tokenIndex = position365, tokenIndex365 - } - l366: - { - position367 := position + position365 := position if !_rules[rulefieldExpr]() { goto l363 } - add(rulePegText, position367) + add(rulePegText, position365) } { add(ruleAction51, position) @@ -3339,153 +3301,175 @@ func (p *PQL) Init(options ...func(*PQL) error) error { }, /* 19 col <- <(( Action52) / (<('\'' singlequotedstring '\'')> Action53) / (<('"' doublequotedstring '"')> Action54))> */ func() bool { - position369, tokenIndex369 := position, tokenIndex + position367, tokenIndex367 := position, tokenIndex { - position370 := position + position368 := position { - position371, tokenIndex371 := position, tokenIndex + position369, tokenIndex369 := position, tokenIndex { - position373 := position + position371 := position if !_rules[ruledigits]() { - goto l372 + goto l370 } - add(rulePegText, position373) + add(rulePegText, position371) } { add(ruleAction52, position) } - goto l371 - l372: - position, tokenIndex = position371, tokenIndex371 + goto l369 + l370: + position, tokenIndex = position369, tokenIndex369 { - position376 := position + position374 := position if buffer[position] != rune('\'') { - goto l375 + goto l373 } position++ if !_rules[rulesinglequotedstring]() { - goto l375 + goto l373 } if buffer[position] != rune('\'') { - goto l375 + goto l373 + } + position++ + add(rulePegText, position374) + } + { + add(ruleAction53, position) + } + goto l369 + l373: + position, tokenIndex = position369, tokenIndex369 + { + position376 := position + if buffer[position] != rune('"') { + goto l367 + } + position++ + if !_rules[ruledoublequotedstring]() { + goto l367 + } + if buffer[position] != rune('"') { + goto l367 } position++ add(rulePegText, position376) } - { - add(ruleAction53, position) - } - goto l371 - l375: - position, tokenIndex = position371, tokenIndex371 - { - position378 := position - if buffer[position] != rune('"') { - goto l369 - } - position++ - if !_rules[ruledoublequotedstring]() { - goto l369 - } - if buffer[position] != rune('"') { - goto l369 - } - position++ - add(rulePegText, position378) - } { add(ruleAction54, position) } } - l371: - add(rulecol, position370) + l369: + add(rulecol, position368) } return true - l369: - position, tokenIndex = position369, tokenIndex369 + l367: + position, tokenIndex = position367, tokenIndex367 return false }, /* 20 row <- <(( Action55) / (<('\'' singlequotedstring '\'')> Action56) / (<('"' doublequotedstring '"')> Action57))> */ nil, /* 21 open <- <('(' sp)> */ + func() bool { + position379, tokenIndex379 := position, tokenIndex + { + position380 := position + if buffer[position] != rune('(') { + goto l379 + } + position++ + if !_rules[rulesp]() { + goto l379 + } + add(ruleopen, position380) + } + return true + l379: + position, tokenIndex = position379, tokenIndex379 + return false + }, + /* 22 close <- <(sp ')' sp)> */ func() bool { position381, tokenIndex381 := position, tokenIndex { position382 := position - if buffer[position] != rune('(') { + if !_rules[rulesp]() { + goto l381 + } + if buffer[position] != rune(')') { goto l381 } position++ if !_rules[rulesp]() { goto l381 } - add(ruleopen, position382) + add(ruleclose, position382) } return true l381: position, tokenIndex = position381, tokenIndex381 return false }, - /* 22 close <- <(sp ')' sp)> */ - func() bool { - position383, tokenIndex383 := position, tokenIndex - { - position384 := position - if !_rules[rulesp]() { - goto l383 - } - if buffer[position] != rune(')') { - goto l383 - } - position++ - if !_rules[rulesp]() { - goto l383 - } - add(ruleclose, position384) - } - return true - l383: - position, tokenIndex = position383, tokenIndex383 - return false - }, /* 23 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position386 := position - l387: + position384 := position + l385: { - position388, tokenIndex388 := position, tokenIndex + position386, tokenIndex386 := position, tokenIndex { - position389, tokenIndex389 := position, tokenIndex + position387, tokenIndex387 := position, tokenIndex if buffer[position] != rune(' ') { - goto l390 - } - position++ - goto l389 - l390: - position, tokenIndex = position389, tokenIndex389 - if buffer[position] != rune('\t') { - goto l391 - } - position++ - goto l389 - l391: - position, tokenIndex = position389, tokenIndex389 - if buffer[position] != rune('\n') { goto l388 } position++ + goto l387 + l388: + position, tokenIndex = position387, tokenIndex387 + if buffer[position] != rune('\t') { + goto l389 + } + position++ + goto l387 + l389: + position, tokenIndex = position387, tokenIndex387 + if buffer[position] != rune('\n') { + goto l386 + } + position++ } - l389: - goto l387 - l388: - position, tokenIndex = position388, tokenIndex388 + l387: + goto l385 + l386: + position, tokenIndex = position386, tokenIndex386 } - add(rulesp, position386) + add(rulesp, position384) } return true }, /* 24 eq <- <(sp '=' sp)> */ + func() bool { + position390, tokenIndex390 := position, tokenIndex + { + position391 := position + if !_rules[rulesp]() { + goto l390 + } + if buffer[position] != rune('=') { + goto l390 + } + position++ + if !_rules[rulesp]() { + goto l390 + } + add(ruleeq, position391) + } + return true + l390: + position, tokenIndex = position390, tokenIndex390 + return false + }, + /* 25 comma <- <(sp ',' sp)> */ func() bool { position392, tokenIndex392 := position, tokenIndex { @@ -3493,302 +3477,298 @@ func (p *PQL) Init(options ...func(*PQL) error) error { if !_rules[rulesp]() { goto l392 } - if buffer[position] != rune('=') { + if buffer[position] != rune(',') { goto l392 } position++ if !_rules[rulesp]() { goto l392 } - add(ruleeq, position393) + add(rulecomma, position393) } return true l392: position, tokenIndex = position392, tokenIndex392 return false }, - /* 25 comma <- <(sp ',' sp)> */ - func() bool { - position394, tokenIndex394 := position, tokenIndex - { - position395 := position - if !_rules[rulesp]() { - goto l394 - } - if buffer[position] != rune(',') { - goto l394 - } - position++ - if !_rules[rulesp]() { - goto l394 - } - add(rulecomma, position395) - } - return true - l394: - position, tokenIndex = position394, tokenIndex394 - return false - }, /* 26 lbrack <- <('[' sp)> */ nil, /* 27 rbrack <- <(sp ']' sp)> */ nil, /* 28 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { - position398, tokenIndex398 := position, tokenIndex + position396, tokenIndex396 := position, tokenIndex { - position399 := position + position397 := position { - position400, tokenIndex400 := position, tokenIndex + position398, tokenIndex398 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l401 + goto l399 } position++ - goto l400 - l401: - position, tokenIndex = position400, tokenIndex400 + goto l398 + l399: + position, tokenIndex = position398, tokenIndex398 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l398 + goto l396 } position++ } + l398: l400: - l402: { - position403, tokenIndex403 := position, tokenIndex + position401, tokenIndex401 := position, tokenIndex { - position404, tokenIndex404 := position, tokenIndex + position402, tokenIndex402 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l405 - } - position++ - goto l404 - l405: - position, tokenIndex = position404, tokenIndex404 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l406 - } - position++ - goto l404 - l406: - position, tokenIndex = position404, tokenIndex404 - if c := buffer[position]; c < rune('0') || c > rune('9') { goto l403 } position++ + goto l402 + l403: + position, tokenIndex = position402, tokenIndex402 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l404 + } + position++ + goto l402 + l404: + position, tokenIndex = position402, tokenIndex402 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l401 + } + position++ } - l404: - goto l402 - l403: - position, tokenIndex = position403, tokenIndex403 + l402: + goto l400 + l401: + position, tokenIndex = position401, tokenIndex401 } - add(ruleIDENT, position399) + add(ruleIDENT, position397) } return true - l398: - position, tokenIndex = position398, tokenIndex398 + l396: + position, tokenIndex = position396, tokenIndex396 return false }, /* 29 digits <- <[0-9]+> */ func() bool { - position407, tokenIndex407 := position, tokenIndex + position405, tokenIndex405 := position, tokenIndex { - position408 := position + position406 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l407 + goto l405 } position++ - l409: + l407: { - position410, tokenIndex410 := position, tokenIndex + position408, tokenIndex408 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l410 + goto l408 } position++ - goto l409 - l410: - position, tokenIndex = position410, tokenIndex410 + goto l407 + l408: + position, tokenIndex = position408, tokenIndex408 } - add(ruledigits, position408) + add(ruledigits, position406) } return true - l407: - position, tokenIndex = position407, tokenIndex407 + l405: + position, tokenIndex = position405, tokenIndex405 return false }, /* 30 signedDigits <- <('-'? digits)> */ nil, /* 31 decimal <- <((signedDigits ('.' digits?)?) / ('-'? '.' digits))> */ func() bool { - position412, tokenIndex412 := position, tokenIndex + position410, tokenIndex410 := position, tokenIndex { - position413 := position + position411 := position { - position414, tokenIndex414 := position, tokenIndex + position412, tokenIndex412 := position, tokenIndex { - position416 := position + position414 := position { - position417, tokenIndex417 := position, tokenIndex + position415, tokenIndex415 := position, tokenIndex if buffer[position] != rune('-') { - goto l417 + goto l415 } position++ - goto l418 - l417: - position, tokenIndex = position417, tokenIndex417 + goto l416 + l415: + position, tokenIndex = position415, tokenIndex415 } - l418: + l416: if !_rules[ruledigits]() { - goto l415 + goto l413 } - add(rulesignedDigits, position416) + add(rulesignedDigits, position414) } { - position419, tokenIndex419 := position, tokenIndex + position417, tokenIndex417 := position, tokenIndex if buffer[position] != rune('.') { - goto l419 + goto l417 } position++ { - position421, tokenIndex421 := position, tokenIndex + position419, tokenIndex419 := position, tokenIndex if !_rules[ruledigits]() { - goto l421 + goto l419 } - goto l422 - l421: - position, tokenIndex = position421, tokenIndex421 + goto l420 + l419: + position, tokenIndex = position419, tokenIndex419 } - l422: - goto l420 - l419: - position, tokenIndex = position419, tokenIndex419 + l420: + goto l418 + l417: + position, tokenIndex = position417, tokenIndex417 } - l420: - goto l414 - l415: - position, tokenIndex = position414, tokenIndex414 + l418: + goto l412 + l413: + position, tokenIndex = position412, tokenIndex412 { - position423, tokenIndex423 := position, tokenIndex + position421, tokenIndex421 := position, tokenIndex if buffer[position] != rune('-') { - goto l423 + goto l421 } position++ - goto l424 - l423: - position, tokenIndex = position423, tokenIndex423 + goto l422 + l421: + position, tokenIndex = position421, tokenIndex421 } - l424: + l422: if buffer[position] != rune('.') { - goto l412 + goto l410 } position++ if !_rules[ruledigits]() { - goto l412 + goto l410 } } - l414: - add(ruledecimal, position413) + l412: + add(ruledecimal, position411) } return true - l412: - position, tokenIndex = position412, tokenIndex412 + l410: + position, tokenIndex = position410, tokenIndex410 return false }, /* 32 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position425, tokenIndex425 := position, tokenIndex + position423, tokenIndex423 := position, tokenIndex { - position426 := position + position424 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l425 + goto l423 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l425 + goto l423 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l425 + goto l423 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l425 + goto l423 } position++ if buffer[position] != rune('-') { - goto l425 + goto l423 } position++ { - position427, tokenIndex427 := position, tokenIndex + position425, tokenIndex425 := position, tokenIndex if buffer[position] != rune('0') { - goto l428 + goto l426 } position++ - goto l427 - l428: - position, tokenIndex = position427, tokenIndex427 + goto l425 + l426: + position, tokenIndex = position425, tokenIndex425 if buffer[position] != rune('1') { - goto l425 + goto l423 } position++ } - l427: + l425: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l425 + goto l423 } position++ if buffer[position] != rune('-') { - goto l425 + goto l423 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l425 + goto l423 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l425 + goto l423 } position++ if buffer[position] != rune('T') { - goto l425 + goto l423 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l425 + goto l423 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l425 + goto l423 } position++ if buffer[position] != rune(':') { - goto l425 + goto l423 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l425 + goto l423 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l425 + goto l423 } position++ - add(ruletimestampbasicfmt, position426) + add(ruletimestampbasicfmt, position424) } return true - l425: - position, tokenIndex = position425, tokenIndex425 + l423: + position, tokenIndex = position423, tokenIndex423 return false }, /* 33 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position429, tokenIndex429 := position, tokenIndex + position427, tokenIndex427 := position, tokenIndex { - position430 := position + position428 := position { - position431, tokenIndex431 := position, tokenIndex + position429, tokenIndex429 := position, tokenIndex if buffer[position] != rune('"') { + goto l430 + } + position++ + { + position431 := position + if !_rules[ruletimestampbasicfmt]() { + goto l430 + } + add(rulePegText, position431) + } + if buffer[position] != rune('"') { + goto l430 + } + position++ + goto l429 + l430: + position, tokenIndex = position429, tokenIndex429 + if buffer[position] != rune('\'') { goto l432 } position++ @@ -3799,45 +3779,27 @@ func (p *PQL) Init(options ...func(*PQL) error) error { } add(rulePegText, position433) } - if buffer[position] != rune('"') { + if buffer[position] != rune('\'') { goto l432 } position++ - goto l431 + goto l429 l432: - position, tokenIndex = position431, tokenIndex431 - if buffer[position] != rune('\'') { - goto l434 - } - position++ + position, tokenIndex = position429, tokenIndex429 { - position435 := position + position434 := position if !_rules[ruletimestampbasicfmt]() { - goto l434 + goto l427 } - add(rulePegText, position435) - } - if buffer[position] != rune('\'') { - goto l434 - } - position++ - goto l431 - l434: - position, tokenIndex = position431, tokenIndex431 - { - position436 := position - if !_rules[ruletimestampbasicfmt]() { - goto l429 - } - add(rulePegText, position436) + add(rulePegText, position434) } } - l431: - add(ruletimestampfmt, position430) + l429: + add(ruletimestampfmt, position428) } return true - l429: - position, tokenIndex = position429, tokenIndex429 + l427: + position, tokenIndex = position427, tokenIndex427 return false }, /* 34 timestamp <- <( Action58)> */ diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 1927a44fe..12b9e4b15 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -522,19 +522,6 @@ func TestPQLDeepEquality(t *testing.T) { {Name: "Row"}, }, }}, - { - name: "TopNwithField=", - call: "TopN(field=myfield, Row(), a=7)", - exp: &Call{ - Name: "TopN", - Args: map[string]interface{}{ - "a": int64(7), - "_field": "myfield", - }, - Children: []*Call{ - {Name: "Row"}, - }, - }}, { name: "RangeEQ", call: "Row(a==7)", From ab41d0492c4343bbb45f44e04328ffc1734571c3 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Thu, 25 Feb 2021 16:04:31 -0600 Subject: [PATCH 16/28] add more tests for TopK, Rows, and SetRowAttrs --- executor_test.go | 9 ++++++ pql/pql.peg | 2 +- pql/pqlpeg_test.go | 72 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/executor_test.go b/executor_test.go index d3b959302..e20d92fe9 100644 --- a/executor_test.go +++ b/executor_test.go @@ -7246,6 +7246,15 @@ toronto,2,11 }, csvVerifier: "pilosa\nzebra\nicecream\n", }, + { + query: "Rows(affinity<0),field=likes)", + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { + if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"pilosa", "zebra", "icecream"}) { + t.Errorf("wrong values: %+v", resp.Results[0]) + } + }, + csvVerifier: "pilosa\nzebra\nicecream\n", + }, { query: "Distinct(Row(affinity>0),field=likes)", qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { diff --git a/pql/pql.peg b/pql/pql.peg index ec6241398..8d7841f5f 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -54,7 +54,7 @@ singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )* fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )* field <- { p.addField(text) } reserved <- '_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field' -posfield <- { p.addPosStr("_field", text) } +posfield <- 'field='? { p.addPosStr("_field", text) } col <- < digits > {p.addPosNum("_col", text)} / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", text)} / < '"' doublequotedstring '"' > {p.addPosStr("_col", text)} diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 12b9e4b15..bbac8337e 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -425,6 +425,54 @@ func TestPQLDeepEquality(t *testing.T) { {Name: "Row", Args: map[string]interface{}{"unicode": "Æ�漢д ☮♬ ♞🜻💣"}}, }, }}, + { + name: "TopK", + call: "TopK(myfield, Row(), a=7)", + exp: &Call{ + Name: "TopK", + Args: map[string]interface{}{ + "a": int64(7), + "_field": "myfield", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, + { + name: "TopKWithField=", + call: "TopK(field=myfield, Row(), a=7)", + exp: &Call{ + Name: "TopK", + Args: map[string]interface{}{ + "a": int64(7), + "_field": "myfield", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, + { + name: "Rows", + call: "Rows(myfield, 9, z=4)", + exp: &Call{ + Name: "Rows", + Args: map[string]interface{}{ + "z": int64(4), + "_field": "myfield", + "_row": int64(9), + }, + }}, + { + name: "RowsWithField=", + call: "Rows(field=myfield, 9, z=4)", + exp: &Call{ + Name: "Rows", + Args: map[string]interface{}{ + "z": int64(4), + "_field": "myfield", + "_row": int64(9), + }, + }}, { name: "SetRowAttrs", call: "SetRowAttrs(myfield, 9, z=4)", @@ -436,6 +484,17 @@ func TestPQLDeepEquality(t *testing.T) { "_row": int64(9), }, }}, + { + name: "SetRowAttrsWithField=", + call: "SetRowAttrs(field=myfield, 9, z=4)", + exp: &Call{ + Name: "SetRowAttrs", + Args: map[string]interface{}{ + "z": int64(4), + "_field": "myfield", + "_row": int64(9), + }, + }}, { name: "SetRowAttrsWithRowKeySingleQuote", call: "SetRowAttrs(myfield, 'rowKey', z=4)", @@ -522,6 +581,19 @@ func TestPQLDeepEquality(t *testing.T) { {Name: "Row"}, }, }}, + { + name: "TopNwithField=", + call: "TopN(field=myfield, Row(), a=7)", + exp: &Call{ + Name: "TopN", + Args: map[string]interface{}{ + "a": int64(7), + "_field": "myfield", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, { name: "RangeEQ", call: "Row(a==7)", From d49a8f953ea028a624fa806327d1563865542feb Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 26 Feb 2021 11:27:39 -0600 Subject: [PATCH 17/28] regenerate pql from modified peg file --- pql/pql.peg.go | 546 ++++++++++++++++++++++++++----------------------- 1 file changed, 292 insertions(+), 254 deletions(-) diff --git a/pql/pql.peg.go b/pql/pql.peg.go index e761e9e60..ed9a8871e 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -8,6 +8,7 @@ import ( "os" "sort" "strconv" + "strings" ) const endSymbol rune = 1114112 @@ -239,7 +240,7 @@ func (node *node32) print(w io.Writer, pretty bool, buffer string) { if !pretty { fmt.Fprintf(w, "%v %v\n", rule, quote) } else { - fmt.Fprintf(w, "\x1B[34m%v\x1B[m %v\n", rule, quote) + fmt.Fprintf(w, "\x1B[36m%v\x1B[m %v\n", rule, quote) } if node.up != nil { print(node.up, depth+1) @@ -414,6 +415,12 @@ func (p *PQL) WriteSyntaxTree(w io.Writer) { p.tokens32.WriteSyntaxTree(w, p.Buffer) } +func (p *PQL) SprintSyntaxTree() string { + var bldr strings.Builder + p.WriteSyntaxTree(&bldr) + return bldr.String() +} + func (p *PQL) Execute() { buffer, _buffer, text, begin, end := p.Buffer, p.buffer, "", 0, 0 for _, token := range p.Tokens() { @@ -3277,17 +3284,48 @@ func (p *PQL) Init(options ...func(*PQL) error) error { }, /* 17 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 18 posfield <- <( Action51)> */ + /* 18 posfield <- <(('f' 'i' 'e' 'l' 'd' '=')? Action51)> */ func() bool { position363, tokenIndex363 := position, tokenIndex { position364 := position { - position365 := position + position365, tokenIndex365 := position, tokenIndex + if buffer[position] != rune('f') { + goto l365 + } + position++ + if buffer[position] != rune('i') { + goto l365 + } + position++ + if buffer[position] != rune('e') { + goto l365 + } + position++ + if buffer[position] != rune('l') { + goto l365 + } + position++ + if buffer[position] != rune('d') { + goto l365 + } + position++ + if buffer[position] != rune('=') { + goto l365 + } + position++ + goto l366 + l365: + position, tokenIndex = position365, tokenIndex365 + } + l366: + { + position367 := position if !_rules[rulefieldExpr]() { goto l363 } - add(rulePegText, position365) + add(rulePegText, position367) } { add(ruleAction51, position) @@ -3301,175 +3339,153 @@ func (p *PQL) Init(options ...func(*PQL) error) error { }, /* 19 col <- <(( Action52) / (<('\'' singlequotedstring '\'')> Action53) / (<('"' doublequotedstring '"')> Action54))> */ func() bool { - position367, tokenIndex367 := position, tokenIndex + position369, tokenIndex369 := position, tokenIndex { - position368 := position + position370 := position { - position369, tokenIndex369 := position, tokenIndex + position371, tokenIndex371 := position, tokenIndex { - position371 := position + position373 := position if !_rules[ruledigits]() { - goto l370 + goto l372 } - add(rulePegText, position371) + add(rulePegText, position373) } { add(ruleAction52, position) } - goto l369 - l370: - position, tokenIndex = position369, tokenIndex369 + goto l371 + l372: + position, tokenIndex = position371, tokenIndex371 { - position374 := position + position376 := position if buffer[position] != rune('\'') { - goto l373 + goto l375 } position++ if !_rules[rulesinglequotedstring]() { - goto l373 + goto l375 } if buffer[position] != rune('\'') { - goto l373 - } - position++ - add(rulePegText, position374) - } - { - add(ruleAction53, position) - } - goto l369 - l373: - position, tokenIndex = position369, tokenIndex369 - { - position376 := position - if buffer[position] != rune('"') { - goto l367 - } - position++ - if !_rules[ruledoublequotedstring]() { - goto l367 - } - if buffer[position] != rune('"') { - goto l367 + goto l375 } position++ add(rulePegText, position376) } + { + add(ruleAction53, position) + } + goto l371 + l375: + position, tokenIndex = position371, tokenIndex371 + { + position378 := position + if buffer[position] != rune('"') { + goto l369 + } + position++ + if !_rules[ruledoublequotedstring]() { + goto l369 + } + if buffer[position] != rune('"') { + goto l369 + } + position++ + add(rulePegText, position378) + } { add(ruleAction54, position) } } - l369: - add(rulecol, position368) + l371: + add(rulecol, position370) } return true - l367: - position, tokenIndex = position367, tokenIndex367 + l369: + position, tokenIndex = position369, tokenIndex369 return false }, /* 20 row <- <(( Action55) / (<('\'' singlequotedstring '\'')> Action56) / (<('"' doublequotedstring '"')> Action57))> */ nil, /* 21 open <- <('(' sp)> */ - func() bool { - position379, tokenIndex379 := position, tokenIndex - { - position380 := position - if buffer[position] != rune('(') { - goto l379 - } - position++ - if !_rules[rulesp]() { - goto l379 - } - add(ruleopen, position380) - } - return true - l379: - position, tokenIndex = position379, tokenIndex379 - return false - }, - /* 22 close <- <(sp ')' sp)> */ func() bool { position381, tokenIndex381 := position, tokenIndex { position382 := position - if !_rules[rulesp]() { - goto l381 - } - if buffer[position] != rune(')') { + if buffer[position] != rune('(') { goto l381 } position++ if !_rules[rulesp]() { goto l381 } - add(ruleclose, position382) + add(ruleopen, position382) } return true l381: position, tokenIndex = position381, tokenIndex381 return false }, + /* 22 close <- <(sp ')' sp)> */ + func() bool { + position383, tokenIndex383 := position, tokenIndex + { + position384 := position + if !_rules[rulesp]() { + goto l383 + } + if buffer[position] != rune(')') { + goto l383 + } + position++ + if !_rules[rulesp]() { + goto l383 + } + add(ruleclose, position384) + } + return true + l383: + position, tokenIndex = position383, tokenIndex383 + return false + }, /* 23 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position384 := position - l385: + position386 := position + l387: { - position386, tokenIndex386 := position, tokenIndex + position388, tokenIndex388 := position, tokenIndex { - position387, tokenIndex387 := position, tokenIndex + position389, tokenIndex389 := position, tokenIndex if buffer[position] != rune(' ') { + goto l390 + } + position++ + goto l389 + l390: + position, tokenIndex = position389, tokenIndex389 + if buffer[position] != rune('\t') { + goto l391 + } + position++ + goto l389 + l391: + position, tokenIndex = position389, tokenIndex389 + if buffer[position] != rune('\n') { goto l388 } position++ - goto l387 - l388: - position, tokenIndex = position387, tokenIndex387 - if buffer[position] != rune('\t') { - goto l389 - } - position++ - goto l387 - l389: - position, tokenIndex = position387, tokenIndex387 - if buffer[position] != rune('\n') { - goto l386 - } - position++ } - l387: - goto l385 - l386: - position, tokenIndex = position386, tokenIndex386 + l389: + goto l387 + l388: + position, tokenIndex = position388, tokenIndex388 } - add(rulesp, position384) + add(rulesp, position386) } return true }, /* 24 eq <- <(sp '=' sp)> */ - func() bool { - position390, tokenIndex390 := position, tokenIndex - { - position391 := position - if !_rules[rulesp]() { - goto l390 - } - if buffer[position] != rune('=') { - goto l390 - } - position++ - if !_rules[rulesp]() { - goto l390 - } - add(ruleeq, position391) - } - return true - l390: - position, tokenIndex = position390, tokenIndex390 - return false - }, - /* 25 comma <- <(sp ',' sp)> */ func() bool { position392, tokenIndex392 := position, tokenIndex { @@ -3477,298 +3493,302 @@ func (p *PQL) Init(options ...func(*PQL) error) error { if !_rules[rulesp]() { goto l392 } - if buffer[position] != rune(',') { + if buffer[position] != rune('=') { goto l392 } position++ if !_rules[rulesp]() { goto l392 } - add(rulecomma, position393) + add(ruleeq, position393) } return true l392: position, tokenIndex = position392, tokenIndex392 return false }, + /* 25 comma <- <(sp ',' sp)> */ + func() bool { + position394, tokenIndex394 := position, tokenIndex + { + position395 := position + if !_rules[rulesp]() { + goto l394 + } + if buffer[position] != rune(',') { + goto l394 + } + position++ + if !_rules[rulesp]() { + goto l394 + } + add(rulecomma, position395) + } + return true + l394: + position, tokenIndex = position394, tokenIndex394 + return false + }, /* 26 lbrack <- <('[' sp)> */ nil, /* 27 rbrack <- <(sp ']' sp)> */ nil, /* 28 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { - position396, tokenIndex396 := position, tokenIndex + position398, tokenIndex398 := position, tokenIndex { - position397 := position + position399 := position { - position398, tokenIndex398 := position, tokenIndex + position400, tokenIndex400 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l399 + goto l401 } position++ - goto l398 - l399: - position, tokenIndex = position398, tokenIndex398 + goto l400 + l401: + position, tokenIndex = position400, tokenIndex400 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l396 + goto l398 } position++ } - l398: l400: + l402: { - position401, tokenIndex401 := position, tokenIndex + position403, tokenIndex403 := position, tokenIndex { - position402, tokenIndex402 := position, tokenIndex + position404, tokenIndex404 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l405 + } + position++ + goto l404 + l405: + position, tokenIndex = position404, tokenIndex404 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l406 + } + position++ + goto l404 + l406: + position, tokenIndex = position404, tokenIndex404 + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l403 } position++ - goto l402 - l403: - position, tokenIndex = position402, tokenIndex402 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l404 - } - position++ - goto l402 - l404: - position, tokenIndex = position402, tokenIndex402 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l401 - } - position++ } - l402: - goto l400 - l401: - position, tokenIndex = position401, tokenIndex401 + l404: + goto l402 + l403: + position, tokenIndex = position403, tokenIndex403 } - add(ruleIDENT, position397) + add(ruleIDENT, position399) } return true - l396: - position, tokenIndex = position396, tokenIndex396 + l398: + position, tokenIndex = position398, tokenIndex398 return false }, /* 29 digits <- <[0-9]+> */ func() bool { - position405, tokenIndex405 := position, tokenIndex + position407, tokenIndex407 := position, tokenIndex { - position406 := position + position408 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l405 + goto l407 } position++ - l407: + l409: { - position408, tokenIndex408 := position, tokenIndex + position410, tokenIndex410 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l408 + goto l410 } position++ - goto l407 - l408: - position, tokenIndex = position408, tokenIndex408 + goto l409 + l410: + position, tokenIndex = position410, tokenIndex410 } - add(ruledigits, position406) + add(ruledigits, position408) } return true - l405: - position, tokenIndex = position405, tokenIndex405 + l407: + position, tokenIndex = position407, tokenIndex407 return false }, /* 30 signedDigits <- <('-'? digits)> */ nil, /* 31 decimal <- <((signedDigits ('.' digits?)?) / ('-'? '.' digits))> */ func() bool { - position410, tokenIndex410 := position, tokenIndex + position412, tokenIndex412 := position, tokenIndex { - position411 := position + position413 := position { - position412, tokenIndex412 := position, tokenIndex + position414, tokenIndex414 := position, tokenIndex { - position414 := position + position416 := position { - position415, tokenIndex415 := position, tokenIndex + position417, tokenIndex417 := position, tokenIndex if buffer[position] != rune('-') { - goto l415 + goto l417 } position++ - goto l416 - l415: - position, tokenIndex = position415, tokenIndex415 + goto l418 + l417: + position, tokenIndex = position417, tokenIndex417 } - l416: + l418: if !_rules[ruledigits]() { - goto l413 + goto l415 } - add(rulesignedDigits, position414) + add(rulesignedDigits, position416) } { - position417, tokenIndex417 := position, tokenIndex + position419, tokenIndex419 := position, tokenIndex if buffer[position] != rune('.') { - goto l417 + goto l419 } position++ { - position419, tokenIndex419 := position, tokenIndex + position421, tokenIndex421 := position, tokenIndex if !_rules[ruledigits]() { - goto l419 + goto l421 } - goto l420 - l419: - position, tokenIndex = position419, tokenIndex419 + goto l422 + l421: + position, tokenIndex = position421, tokenIndex421 } - l420: - goto l418 - l417: - position, tokenIndex = position417, tokenIndex417 + l422: + goto l420 + l419: + position, tokenIndex = position419, tokenIndex419 } - l418: - goto l412 - l413: - position, tokenIndex = position412, tokenIndex412 + l420: + goto l414 + l415: + position, tokenIndex = position414, tokenIndex414 { - position421, tokenIndex421 := position, tokenIndex + position423, tokenIndex423 := position, tokenIndex if buffer[position] != rune('-') { - goto l421 + goto l423 } position++ - goto l422 - l421: - position, tokenIndex = position421, tokenIndex421 + goto l424 + l423: + position, tokenIndex = position423, tokenIndex423 } - l422: + l424: if buffer[position] != rune('.') { - goto l410 + goto l412 } position++ if !_rules[ruledigits]() { - goto l410 + goto l412 } } - l412: - add(ruledecimal, position411) + l414: + add(ruledecimal, position413) } return true - l410: - position, tokenIndex = position410, tokenIndex410 + l412: + position, tokenIndex = position412, tokenIndex412 return false }, /* 32 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position423, tokenIndex423 := position, tokenIndex + position425, tokenIndex425 := position, tokenIndex { - position424 := position + position426 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if buffer[position] != rune('-') { - goto l423 + goto l425 } position++ { - position425, tokenIndex425 := position, tokenIndex + position427, tokenIndex427 := position, tokenIndex if buffer[position] != rune('0') { - goto l426 + goto l428 } position++ - goto l425 - l426: - position, tokenIndex = position425, tokenIndex425 + goto l427 + l428: + position, tokenIndex = position427, tokenIndex427 if buffer[position] != rune('1') { - goto l423 + goto l425 } position++ } - l425: + l427: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if buffer[position] != rune('-') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if buffer[position] != rune('T') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if buffer[position] != rune(':') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l423 + goto l425 } position++ - add(ruletimestampbasicfmt, position424) + add(ruletimestampbasicfmt, position426) } return true - l423: - position, tokenIndex = position423, tokenIndex423 + l425: + position, tokenIndex = position425, tokenIndex425 return false }, /* 33 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position427, tokenIndex427 := position, tokenIndex + position429, tokenIndex429 := position, tokenIndex { - position428 := position + position430 := position { - position429, tokenIndex429 := position, tokenIndex + position431, tokenIndex431 := position, tokenIndex if buffer[position] != rune('"') { - goto l430 - } - position++ - { - position431 := position - if !_rules[ruletimestampbasicfmt]() { - goto l430 - } - add(rulePegText, position431) - } - if buffer[position] != rune('"') { - goto l430 - } - position++ - goto l429 - l430: - position, tokenIndex = position429, tokenIndex429 - if buffer[position] != rune('\'') { goto l432 } position++ @@ -3779,27 +3799,45 @@ func (p *PQL) Init(options ...func(*PQL) error) error { } add(rulePegText, position433) } - if buffer[position] != rune('\'') { + if buffer[position] != rune('"') { goto l432 } position++ - goto l429 + goto l431 l432: - position, tokenIndex = position429, tokenIndex429 + position, tokenIndex = position431, tokenIndex431 + if buffer[position] != rune('\'') { + goto l434 + } + position++ { - position434 := position + position435 := position if !_rules[ruletimestampbasicfmt]() { - goto l427 + goto l434 } - add(rulePegText, position434) + add(rulePegText, position435) + } + if buffer[position] != rune('\'') { + goto l434 + } + position++ + goto l431 + l434: + position, tokenIndex = position431, tokenIndex431 + { + position436 := position + if !_rules[ruletimestampbasicfmt]() { + goto l429 + } + add(rulePegText, position436) } } - l429: - add(ruletimestampfmt, position428) + l431: + add(ruletimestampfmt, position430) } return true - l427: - position, tokenIndex = position427, tokenIndex427 + l429: + position, tokenIndex = position429, tokenIndex429 return false }, /* 34 timestamp <- <( Action58)> */ From 90278b206cb37b7333b396504f01006d24b55e0b Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 26 Feb 2021 10:35:05 -0700 Subject: [PATCH 18/28] Add benchmarking of autogenerated ID data --- scripts/bench_write.sh | 2 +- scripts/etc/gloat/gh.issues.autogenerate.yml | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 scripts/etc/gloat/gh.issues.autogenerate.yml diff --git a/scripts/bench_write.sh b/scripts/bench_write.sh index a5306debc..5757a953d 100755 --- a/scripts/bench_write.sh +++ b/scripts/bench_write.sh @@ -21,7 +21,7 @@ SHA=$(git -C $PILOSA_SRC rev-parse HEAD) # Format current date. DATE=$(date '+%Y%m%d') -for FILENAME in gh.1m.yml gh.issues.keyed.yml gh.issues.unkeyed.yml +for FILENAME in gh.1m.yml gh.issues.keyed.yml gh.issues.unkeyed.yml gh.issues.autogenerate.yml do WORKFLOW_PATH="${BASH_SOURCE%/*}/etc/gloat/${FILENAME}" WORKFLOW_NAME="$(gloat workflow name $WORKFLOW_PATH)" diff --git a/scripts/etc/gloat/gh.issues.autogenerate.yml b/scripts/etc/gloat/gh.issues.autogenerate.yml new file mode 100644 index 000000000..d1ceeace5 --- /dev/null +++ b/scripts/etc/gloat/gh.issues.autogenerate.yml @@ -0,0 +1,9 @@ +name: "GitHub Issues Import Load Testing (two weeks, autogenerated ID)" + +main: "pilosa server --data-dir ${TMPDIR} --txsrc ${STORAGE_BACKEND}" +load: "molecula-consumer-github -i issues -a --external-generate --record-type issue --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-13T23:00:00Z --cache-dir ~/.githubarchive" + +health_url: "http://localhost:10101/status" +vars_urls: + - http://localhost:10101/debug/vars + - http://localhost:7070/debug/vars From 6f7676784197ce674f7d80c018ff2a1f47e314cd Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 26 Feb 2021 12:06:25 -0600 Subject: [PATCH 19/28] add test in executor.go to compare results of query w/without field= --- executor_test.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/executor_test.go b/executor_test.go index e20d92fe9..33f43a060 100644 --- a/executor_test.go +++ b/executor_test.go @@ -7246,15 +7246,6 @@ toronto,2,11 }, csvVerifier: "pilosa\nzebra\nicecream\n", }, - { - query: "Rows(affinity<0),field=likes)", - qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { - if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"pilosa", "zebra", "icecream"}) { - t.Errorf("wrong values: %+v", resp.Results[0]) - } - }, - csvVerifier: "pilosa\nzebra\nicecream\n", - }, { query: "Distinct(Row(affinity>0),field=likes)", qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { @@ -7414,6 +7405,20 @@ pangolin,1,100 query: "GroupBy(Rows(field=dinner), sort=\"count desc\", limit=2)", csvVerifier: `chinese,3 pizza,2 +`, + }, + { + query: "TopK(dinner)", + csvVerifier: `chinese,3 +pizza,2 +leftovers,1 +`, + }, + { + query: "TopK(field=dinner)", + csvVerifier: `chinese,3 +pizza,2 +leftovers,1 `, }, } From 8bc8368c578bbfef22d66fa8ea845db78e3036d8 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 26 Feb 2021 12:58:59 -0600 Subject: [PATCH 20/28] change test arguments to be appropriate --- pql/ast.go | 15 ++++++++++++--- pql/pqlpeg_test.go | 14 ++++++-------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index a4b80f4ae..9c844fa5c 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -63,7 +63,11 @@ func (q *Query) lastCallStackElem() *callStackElem { } func (q *Query) addPosNum(key, value string) { - q.addField(key) + if key == "field" { + q.addField("_field") + } else { + q.addField(key) + } q.addNumVal(value) } @@ -431,8 +435,13 @@ var callInfoByFunc = map[string]callInfo{ }, }, - // things that take _field - "TopN": allowUnderField, + "TopN": { + allowUnknown: true, + prototypes: map[string]interface{}{ + "_field": "", + "field": "", + }, + }, // special cases: "Clear": { allowUnknown: true, diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index bbac8337e..da49b8c39 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -427,12 +427,12 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "TopK", - call: "TopK(myfield, Row(), a=7)", + call: "TopK(myfield, Row()), k=7", exp: &Call{ Name: "TopK", Args: map[string]interface{}{ - "a": int64(7), "_field": "myfield", + "k": int64(7), }, Children: []*Call{ {Name: "Row"}, @@ -440,12 +440,12 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "TopKWithField=", - call: "TopK(field=myfield, Row(), a=7)", + call: "TopK(field=myfield, Row(), k=7)", exp: &Call{ Name: "TopK", Args: map[string]interface{}{ - "a": int64(7), "_field": "myfield", + "k": int64(7), }, Children: []*Call{ {Name: "Row"}, @@ -453,24 +453,22 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "Rows", - call: "Rows(myfield, 9, z=4)", + call: "Rows(myfield, z=4)", exp: &Call{ Name: "Rows", Args: map[string]interface{}{ "z": int64(4), "_field": "myfield", - "_row": int64(9), }, }}, { name: "RowsWithField=", - call: "Rows(field=myfield, 9, z=4)", + call: "Rows(field=myfield, z=4)", exp: &Call{ Name: "Rows", Args: map[string]interface{}{ "z": int64(4), "_field": "myfield", - "_row": int64(9), }, }}, { From dca19f7e9466a7b73512453eef0f0cbb997d014e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 9 Feb 2021 14:32:26 -0600 Subject: [PATCH 21/28] Add new Docker build process and continuous delivery --- .circleci/config.yml | 75 ++++++++++++++---------- .dockerignore | 6 +- Dockerfile | 41 +++++++++++--- Makefile | 132 +++++++++++++++++++++++++------------------ lattice | 2 +- server/config.go | 5 +- server/default.go | 24 -------- server/release.go | 24 -------- 8 files changed, 163 insertions(+), 146 deletions(-) delete mode 100644 server/default.go delete mode 100644 server/release.go diff --git a/.circleci/config.yml b/.circleci/config.yml index ef6b7ce80..3026f2e27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ executors: parameters: version: type: string - default: "1.15.5" + default: "1.15.8" resource_class: type: string default: medium @@ -17,7 +17,8 @@ executors: commands: add-github-auth: steps: - - run: git config --global url."https://moleculacorp:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com".insteadOf "https://github.com" + - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/" + - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:" restore-mod-cache: steps: - restore_cache: @@ -67,7 +68,7 @@ jobs: name: golang steps: - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request - - run: curl https://moleculacorp:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/pilosa/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e + - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/pilosa/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e test-build-arm: executor: name: golang @@ -84,7 +85,7 @@ jobs: default: medium golang_version: type: string - default: "1.15.5" + default: "1.15.8" shard_width: type: string default: "20" @@ -114,17 +115,6 @@ jobs: - checkout-plus - setup_remote_docker - run: make clustertests-build - prerelease: - executor: - name: golang - steps: - - checkout-plus - - run: make prerelease - - store_artifacts: - path: build - - persist_to_workspace: - root: . - paths: build release: executor: name: golang @@ -132,28 +122,32 @@ jobs: - checkout-plus - attach_workspace: at: . - - run: make release + - setup_remote_docker: + version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 + - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin + - run: make docker-release - store_artifacts: path: build - persist_to_workspace: root: . paths: build - prerelease-upload: - docker: - - image: circleci/python:2.7-jessie + publish_release: + executor: + name: golang steps: - - checkout-plus - attach_workspace: at: . - - run: sudo pip install awscli - - run: make prerelease-upload + - run: go get github.com/tcnksm/ghr + - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/ docker-build: executor: name: golang steps: - checkout-plus - - setup_remote_docker - - run: make docker + - setup_remote_docker: + version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 + - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin + - run: make docker GO_VERSION=1.15.8 - run: docker run pilosa:$(git describe --tags) help dockerhub-upload-unstable: executor: @@ -163,7 +157,7 @@ jobs: - setup_remote_docker - run: make docker - run: docker run pilosa:$(git describe --tags) help - - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >> dockerhub-upload-stable: executor: @@ -173,7 +167,7 @@ jobs: - setup_remote_docker - run: make docker - run: docker run pilosa:$(git describe --tags) help - - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >> - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest @@ -186,12 +180,15 @@ workflows: tags: only: /^v.*/ - linter: + context: molecula requires: - setup - check-license-headers: + context: molecula requires: - setup - go-mod-tidy: + context: molecula requires: - setup - check-changelog-label: @@ -199,6 +196,7 @@ workflows: requires: - setup - test-build-arm: + context: molecula requires: - setup - test: @@ -209,7 +207,7 @@ workflows: - setup matrix: parameters: - golang_version: ["1.14.12", "1.15.5"] + golang_version: ["1.14.15", "1.15.8"] - test: name: << matrix.test_make_target >> resource_class: xlarge @@ -221,22 +219,37 @@ workflows: test_make_target: ["test-race", "test-txstore-rbf", "test-txstore-rbf_bolt"] - test: name: test-shardwidth-22 + context: molecula shard_width: "22" resource_class: large requires: - setup - cluster-tests: + context: molecula requires: - setup - docker-build: context: molecula requires: - setup - - prerelease: + - release: + context: molecula requires: - - linter - - check-license-headers - - test-golang-1.15.5 + - setup + filters: + tags: + only: /^v.*/ + branches: + ignore: /.*/ + - publish_release: + context: molecula + requires: + - release + filters: + tags: + only: /^v.*/ + branches: + ignore: /.*/ - dockerhub-upload-unstable: context: molecula requires: diff --git a/.dockerignore b/.dockerignore index d5de893f3..0e869cde0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,5 @@ -lattice +lattice/.git +lattice/node_modules +lattice/build +statik/statik.go +build diff --git a/Dockerfile b/Dockerfile index 955f499ed..00d2c7686 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,46 @@ -FROM golang:1.14.10 as builder +ARG GO_VERSION=latest -ARG BUILD_FLAGS +####################### +### Lattice builder ### +####################### + +FROM moleculacorp/nodejs:latest as lattice-builder +WORKDIR /lattice + +COPY lattice/package.json ./ +COPY lattice/yarn.lock ./ +RUN yarn install + +COPY lattice ./ +RUN yarn build + +###################### +### Pilosa builder ### +###################### + +FROM golang:${GO_VERSION} as pilosa-builder ARG MAKE_FLAGS +WORKDIR /pilosa -COPY . pilosa +RUN go get github.com/rakyll/statik -RUN cd pilosa && make install FLAGS="-a -mod=vendor ${BUILD_FLAGS}" ${MAKE_FLAGS} +COPY . ./ +COPY --from=lattice-builder /lattice/build /lattice +RUN /go/bin/statik -src=/lattice -dest=/pilosa -FROM alpine:3.12.1 +RUN make build ${MAKE_FLAGS} -LABEL maintainer "dev@pilosa.com" +##################### +### Pilosa runner ### +##################### + +FROM alpine:3.13.2 as runner + +LABEL maintainer "dev@molecula.com" RUN apk add --no-cache curl jq -COPY --from=builder /go/bin/pilosa /pilosa +COPY --from=pilosa-builder /pilosa/build/pilosa / COPY LICENSE /LICENSE COPY NOTICE /NOTICE diff --git a/Makefile b/Makefile index 0f06e1b92..07f96df0a 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,24 @@ -.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice CLONE_URL=github.com/pilosa/pilosa MOD_VERSION=v2 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null) VARIANT = Molecula -VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH) -BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) +GO=go +GOOS=$(shell $(GO) env GOOS) +GOARCH=$(shell $(GO) env GOARCH) +BINOUT=build +FLAGS=-o $(BINOUT)/pilosa +VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH) +BRANCH := $(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)) BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT) -X github.com/pilosa/pilosa/v2.TrialDeadline=$(TRIAL_DEADLINE)" -TRIAL_STRING = $(if $(TRIAL_DEADLINE),"-trial-$(TRIAL_DEADLINE)","") -GO_VERSION=1.14.10 -RELEASE ?= 0 -RELEASE_ENABLED = $(subst 0,,$(RELEASE)) -BUILD_TAGS += $(if $(RELEASE_ENABLED),release) +GO_VERSION=1.15.8 +DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release BUILD_TAGS += shardwidth$(SHARD_WIDTH) TEST_TAGS = roaringparanoia define LICENSE_HASH_CODE @@ -43,15 +45,15 @@ clean: # Set up vendor directory using `go mod vendor` vendor: go.mod - go mod vendor + $(GO) mod vendor # Run test suite test: - go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v + $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v # Run test suite with race flag test-race: - CGO_ENABLED=1 go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout 60m -v + CGO_ENABLED=1 $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout 60m -v testv: topt testvsub @@ -66,7 +68,7 @@ testvsub: set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i"; \ cd $$i; pwd; \ - go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \ + $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \ echo; echo "999 done testing subpkg $$i"; \ cd ..; \ done @@ -75,7 +77,7 @@ testvsub-race: set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i -race"; \ cd $$i; pwd; \ - CGO_ENABLED=1 go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \ + CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \ echo; echo "999 done testing subpkg $$i -race"; \ cd ..; \ done @@ -84,7 +86,7 @@ tour: ./tournament.sh bench: - go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS) + $(GO) test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS) # Run test suite with coverage enabled cover: @@ -93,18 +95,18 @@ cover: # Run test suite with coverage enabled and view coverage results in browser cover-viz: cover - go tool cover -html=build/coverage.out + $(GO) tool cover -html=build/coverage.out # Compile Pilosa build: - go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa + $(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa # Create a single release build under the build directory release-build: - $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa$(TRIAL_STRING)-$(VERSION_ID)/pilosa" RELEASE=1 - cp NOTICE README.md LICENSE build/pilosa$(TRIAL_STRING)-$(VERSION_ID) - tar -cvz -C build -f build/pilosa$(TRIAL_STRING)-$(VERSION_ID).tar.gz pilosa$(TRIAL_STRING)-$(VERSION_ID)/ - @echo Created release build: build/pilosa$(TRIAL_STRING)-$(VERSION_ID).tar.gz + $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" + cp NOTICE README.md LICENSE build/pilosa$(VERSION_ID) + tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ + @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz # Error out if there are untracked changes in Git check-clean: @@ -112,16 +114,16 @@ ifndef SKIP_CHECK_CLEAN $(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.)) endif -# Create release build tarballs for all supported platforms. Linux compilation happens under Docker. -release: check-clean generate-statik +# Create release build tarballs for all supported platforms. DEPRECATED: Use `docker-release` +release: check-clean generate-statik-docker $(MAKE) release-build GOOS=darwin GOARCH=amd64 - $(MAKE) release-build GOOS=linux GOARCH=amd64 $(if $(IS_MACOS),DOCKER_BUILD=1) + $(MAKE) release-build GOOS=linux GOARCH=amd64 # Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI. release-sans-ui: check-clean rm -f statik/statik.go $(MAKE) release-build GOOS=darwin GOARCH=amd64 - $(MAKE) release-build GOOS=linux GOARCH=amd64 $(if $(IS_MACOS),DOCKER_BUILD=1) + $(MAKE) release-build GOOS=linux GOARCH=amd64 # try (e.g.) internal/clustertests/docker-compose-replication2.yml DOCKER_COMPOSE=internal/clustertests/docker-compose.yml @@ -141,28 +143,21 @@ clustertests-build: vendor docker-compose -f $(DOCKER_COMPOSE) down -v docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build -# Create prerelease builds -prerelease: - $(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\) - $(if $(shell git describe --tags --exact-match HEAD),$(MAKE) release) - -prerelease-upload: - aws s3 sync build/ s3://build.pilosa.com/ --exclude "*" --include "*.tar.gz" --acl public-read - # Install Pilosa install: - go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa install-bench: - go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench # Ensure lattice is cloned and the pinned version is checked out lattice: git submodule update --init # Build the lattice assets -build-lattice: lattice require-yarn - cd lattice && yarn install && yarn build +build-lattice: lattice + docker build -t lattice:build ./lattice + export LATTICE=`docker create lattice:build`; docker cp $$LATTICE:/lattice/. ./lattice/build && docker rm $$LATTICE # Upgrade lattice to the latest version upgrade-lattice: lattice @@ -170,15 +165,19 @@ upgrade-lattice: lattice # `go generate` protocol buffers generate-protoc: require-protoc require-protoc-gen-gofast - go generate github.com/pilosa/pilosa/v2/internal + $(GO) generate github.com/pilosa/pilosa/v2/internal # `go generate` statik assets (lattice UI) generate-statik: build-lattice require-statik - go generate github.com/pilosa/pilosa/v2/statik + $(GO) generate github.com/pilosa/pilosa/v2/statik + +# `go generate` statik assets (lattice UI) in Docker +generate-statik-docker: build-lattice + docker run --rm -t -v $(PWD):/pilosa golang:1.15.8 sh -c "go get github.com/rakyll/statik && /go/bin/statik -src=/pilosa/lattice/build -dest=/pilosa -f" # `go generate` stringers generate-stringer: - go generate github.com/pilosa/pilosa/v2 + $(GO) generate github.com/pilosa/pilosa/v2 generate-pql: require-peg cd pql && peg -inline pql.peg && cd .. @@ -195,28 +194,49 @@ generate-proto-grpc: require-protoc require-protoc-gen-go # `go generate` all needed packages generate: generate-protoc generate-statik generate-stringer generate-pql +# Create release using Docker +docker-release: + $(MAKE) docker-build GOOS=linux GOARCH=amd64 + $(MAKE) docker-build GOOS=darwin GOARCH=amd64 + +# Build a release in Docker +docker-build: vendor lattice + docker build \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \ + --target pilosa-builder \ + --tag pilosa:build . + docker create --name pilosa-build pilosa:build + mkdir -p build/pilosa-$(VERSION_ID) + docker cp pilosa-build:/pilosa/build/. ./build/pilosa-$(VERSION_ID) + cp NOTICE LICENSE ./build/pilosa-$(VERSION_ID) + docker rm pilosa-build + tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ + # Create Docker image from Dockerfile -docker: vendor - docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" . +docker-image: vendor lattice + docker build \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \ + --tag pilosa:$(VERSION) . @echo Created docker image: pilosa:$(VERSION) +# Create docker image (alias) +docker: docker-image # alias + # Tag and push a Docker image docker-tag-push: vendor docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET) docker push $(DOCKER_TARGET) @echo Pushed docker image: $(DOCKER_TARGET) -# Compile Pilosa inside Docker container -docker-build: vendor - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) make build FLAGS="$(FLAGS) -mod=vendor" RELEASE=$(RELEASE) - # Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory. pilosa-keydump: - go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump # Install diagnostic pilosa-chk tool for string translations and fragment checksums. pilosa-chk: - go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk pilosa-fsck: cd ./cmd/pilosa-fsck && make install && make release @@ -230,13 +250,13 @@ docker-test: # The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt. topt: mv log.topt.roar log.topt.roar.prev || true - $(eval SHELL:=/bin/bash) set -o pipefail; go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar + $(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar @echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l @echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l topt-race: mv log.topt.race log.topt.race.prev || true - $(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race + $(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race @echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l @echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l @@ -295,27 +315,27 @@ install-statik: go get -u github.com/rakyll/statik install-stringer: - GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer + GO111MODULE=off $(GO) get -u golang.org/x/tools/cmd/stringer install-protoc-gen-gofast: - GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast + GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast install-protoc-gen-go: - GO111MODULE=off go get -u github.com/golang/protobuf/protoc-gen-go + GO111MODULE=off $(GO) get -u github.com/golang/protobuf/protoc-gen-go install-protoc: @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html install-peg: - GO111MODULE=off go get github.com/pointlander/peg + GO111MODULE=off $(GO) get github.com/pointlander/peg install-golangci-lint: - GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint + GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint install-gometalinter: - GO111MODULE=off go get -u github.com/alecthomas/gometalinter + GO111MODULE=off $(GO) get -u github.com/alecthomas/gometalinter GO111MODULE=off gometalinter --install - GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode + GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode test-txstore-rbf: PILOSA_TXSRC=rbf $(MAKE) testv-race diff --git a/lattice b/lattice index fa773628a..a7c05f2f6 160000 --- a/lattice +++ b/lattice @@ -1 +1 @@ -Subproject commit fa773628a276e2590785a87fbc236c7e88ea6284 +Subproject commit a7c05f2f6aa59d9723403f49de72f5ef0682018a diff --git a/server/config.go b/server/config.go index 59f390210..2a0e3aad9 100644 --- a/server/config.go +++ b/server/config.go @@ -31,8 +31,9 @@ import ( ) const ( - defaultBindPort = "10101" - defaultBindGRPCPort = "20101" + defaultBindPort = "10101" + defaultBindGRPCPort = "20101" + defaultDiagnosticsInterval = 1 * time.Hour ) // TLSConfig contains TLS configuration diff --git a/server/default.go b/server/default.go deleted file mode 100644 index ce2fe8aaa..000000000 --- a/server/default.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// +build !release -// -// This file sets defaults to be overridden by release.go - -package server - -import "time" - -// defaultDiagnosticsInterval is the default sync frequency diagnostic metrics. A value of 0 disables diagnostics. -const defaultDiagnosticsInterval = time.Duration(0) diff --git a/server/release.go b/server/release.go deleted file mode 100644 index d988f4f81..000000000 --- a/server/release.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// +build release -// -// This file sets release-specific variables. - -package server - -import "time" - -// defaultDiagnosticsInterval is the default sync frequency diagnostic metrics. -const defaultDiagnosticsInterval = 1 * time.Hour From 441630d804ff5013f8b2653ea48feaa5a0cac021 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 26 Feb 2021 16:13:00 -0600 Subject: [PATCH 22/28] try fixing tests --- pql/pqlpeg_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index da49b8c39..b54784926 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -427,7 +427,7 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "TopK", - call: "TopK(myfield, Row()), k=7", + call: "TopK(myfield, Row(), k=7)", exp: &Call{ Name: "TopK", Args: map[string]interface{}{ @@ -453,7 +453,7 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "Rows", - call: "Rows(myfield, z=4)", + call: "Rows(myfield)", exp: &Call{ Name: "Rows", Args: map[string]interface{}{ @@ -463,7 +463,7 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RowsWithField=", - call: "Rows(field=myfield, z=4)", + call: "Rows(field=myfield)", exp: &Call{ Name: "Rows", Args: map[string]interface{}{ From 3822e38952e46aa9b8f1d2a48a3aeae7ce6c194d Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 26 Feb 2021 16:15:37 -0600 Subject: [PATCH 23/28] remove z arg from Rows tests --- pql/pqlpeg_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index b54784926..9d3d1192e 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -457,7 +457,6 @@ func TestPQLDeepEquality(t *testing.T) { exp: &Call{ Name: "Rows", Args: map[string]interface{}{ - "z": int64(4), "_field": "myfield", }, }}, @@ -467,7 +466,6 @@ func TestPQLDeepEquality(t *testing.T) { exp: &Call{ Name: "Rows", Args: map[string]interface{}{ - "z": int64(4), "_field": "myfield", }, }}, From 23f44725cb6c549777be3414cb554704bb6222c7 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Fri, 26 Feb 2021 16:36:16 -0600 Subject: [PATCH 24/28] remove var allowUnderField --- pql/ast.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index 9c844fa5c..fe8a8cb2e 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -332,13 +332,6 @@ type stringOrInt64Type struct{} var stringOrInt64 stringOrInt64Type -var allowUnderField = callInfo{ - allowUnknown: true, - prototypes: map[string]interface{}{ - "_field": "", - }, -} - var allowField = callInfo{ allowUnknown: false, prototypes: map[string]interface{}{ From 15aa7d1ad8d922b618bf64de5a11039dd0767dcc Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Mon, 1 Mar 2021 11:18:31 -0600 Subject: [PATCH 25/28] add field to SetRowAttrs and TopK to prototypes in ast.go --- pql/ast.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pql/ast.go b/pql/ast.go index fe8a8cb2e..a889c9489 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -421,6 +421,7 @@ var callInfoByFunc = map[string]callInfo{ allowUnknown: false, prototypes: map[string]interface{}{ "_field": "", + "field": "", "k": int64(0), "filter": nil, "from": nil, @@ -483,6 +484,7 @@ var callInfoByFunc = map[string]callInfo{ allowUnknown: true, prototypes: map[string]interface{}{ "_field": "", + "field": "", "_row": stringOrInt64, }, }, From bfa9682ba57d68aeba51108a4a7377afcc2f60be Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 1 Mar 2021 17:01:06 -0600 Subject: [PATCH 26/28] Fix incorrect build flags --- Dockerfile | 2 +- Makefile | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 00d2c7686..20170e420 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ COPY . ./ COPY --from=lattice-builder /lattice/build /lattice RUN /go/bin/statik -src=/lattice -dest=/pilosa -RUN make build ${MAKE_FLAGS} +RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS} ##################### ### Pilosa runner ### diff --git a/Makefile b/Makefile index 07f96df0a..70c94aaf3 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,6 @@ VARIANT = Molecula GO=go GOOS=$(shell $(GO) env GOOS) GOARCH=$(shell $(GO) env GOARCH) -BINOUT=build -FLAGS=-o $(BINOUT)/pilosa VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH) BRANCH := $(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)) BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) From a94c6cf8a23225d1dc98d8f35cf220fbe7072948 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 2 Mar 2021 13:43:27 -0600 Subject: [PATCH 27/28] Docker login prior to building image --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3026f2e27..5a0dbdd8d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,9 +155,9 @@ jobs: steps: - checkout-plus - setup_remote_docker + - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker - run: docker run pilosa:$(git describe --tags) help - - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >> dockerhub-upload-stable: executor: @@ -165,9 +165,9 @@ jobs: steps: - checkout-plus - setup_remote_docker + - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker - run: docker run pilosa:$(git describe --tags) help - - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >> - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest From 0532987d57e25cfdd8cfaa072f87bfc5bcc09237 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 2 Mar 2021 14:22:17 -0600 Subject: [PATCH 28/28] Workaround CI versioning issue --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a0dbdd8d..266d5da25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -154,7 +154,8 @@ jobs: name: golang steps: - checkout-plus - - setup_remote_docker + - setup_remote_docker: + version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker - run: docker run pilosa:$(git describe --tags) help @@ -164,7 +165,8 @@ jobs: name: golang steps: - checkout-plus - - setup_remote_docker + - setup_remote_docker: + version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker - run: docker run pilosa:$(git describe --tags) help