diff --git a/api.go b/api.go index 224c6bf15..f67ecf100 100644 --- a/api.go +++ b/api.go @@ -431,8 +431,15 @@ func importWorker(importWork chan importJob) { } if j.req.UpdateExistence { if ef := j.field.idx.existenceField(); ef != nil { - existence := combineForExistence(data) - ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false) + existence, err := combineForExistence(data) + if err != nil { + return errors.Wrap(err, "merging existence on roaring import") + } + + err = ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false) + if err != nil { + return errors.Wrap(err, "updating existence on roaring import") + } } } @@ -458,19 +465,21 @@ func importWorker(importWork chan importJob) { } // merge all rows to singled existence row -func combineForExistence(inputRoaringData []byte) []byte { +func combineForExistence(inputRoaringData []byte) ([]byte, error) { rowSize := uint64(1 << shardVsContainerExponent) rit, err := roaring.NewRoaringIterator(inputRoaringData) if err != nil { - panicOn(err) + return nil, err } bm := roaring.NewBitmap() - bm.MergeRoaringRawIteratorIntoExists(rit, rowSize) + err = bm.MergeRoaringRawIteratorIntoExists(rit, rowSize) + if err != nil { + return nil, err + } buf := new(bytes.Buffer) _, err = bm.WriteTo(buf) - panicOn(err) - return buf.Bytes() + return buf.Bytes(), err } // ImportRoaring is a low level interface for importing data to Pilosa when diff --git a/cmd/sauron/design.org b/cmd/sauron/design.org deleted file mode 100644 index aa579ab34..000000000 --- a/cmd/sauron/design.org +++ /dev/null @@ -1,199 +0,0 @@ - -* Oracle Design -- setup server that intersepts PQL queries and proxies to live pilosa server -- implement all the pilosa functions using SQL, specifically the sqlite flavor. This will - provide validation against pilosa results -- provide timings for each query path, PQL and SQL and compare -- this is intended to be a correctness check and not a performance check. the - sql solution is likely to be pretty slow with realitively small numbers ~1millions - #+BEGIN_SRC - .─────. - ,' `. ┌────────────┐ ┌────────┐ -;pilosa json: │ Sauron │ ┌──▶│ pilosa │ -: client ;──────────▶ │ │───┤ └────────┘ - ╲ ╱ │ │ │ ┌────────┐ - `. ,' └────────────┘ └──▶│ sqlite │ - `───' └────────┘ - #+END_SRC -* Setup Sql (sqlite) -#+name: setup test -#+header: :results silent -#+header: :db poc.db - #+BEGIN_SRC sqlite - create table bits (field string,row string, column bigint); - create table bsi (field string,column bigint , val bigint); - CREATE VIEW columns as - select distinct column from bits - UNION - select distinct column from bsi; - - INSERT INTO bits VALUES( 'color','red',1); - INSERT INTO bits VALUES( 'color','red',2); - INSERT INTO bits VALUES( 'color','red',3); - INSERT INTO bits VALUES( 'color','red',4); - INSERT INTO bits VALUES( 'color','red',5); - INSERT INTO bits VALUES( 'color','green',2); - INSERT INTO bits VALUES( 'color','green',4); - INSERT INTO bits VALUES( 'color','green',6); - INSERT INTO bits VALUES( 'color','green',8); - INSERT INTO bits VALUES( 'color','yellow',1); - INSERT INTO bits VALUES( 'color','yellow',3); - INSERT INTO bits VALUES( 'color','yellow',5); - INSERT INTO bits VALUES( 'color','yellow',7); - INSERT INTO bits VALUES( 'color','orange',5); - INSERT INTO bits VALUES( 'color','orange',6); - INSERT INTO bits VALUES( 'color','orange',7); - INSERT INTO bits VALUES( 'color','orange',8); - - INSERT INTO bsi VALUES( 'luminosity',1, 1); - INSERT INTO bsi VALUES( 'luminosity',2, 2); - INSERT INTO bsi VALUES( 'luminosity',3, 4); - INSERT INTO bsi VALUES( 'luminosity',4, 5); - INSERT INTO bsi VALUES( 'luminosity',5, 4); - INSERT INTO bsi VALUES( 'luminosity',6, 3); - INSERT INTO bsi VALUES( 'luminosity',7, 2); - INSERT INTO bsi VALUES( 'luminosity',8, 1); - #+END_SRC - -* TODO PQL to SQL TODO [10/11] -- [X] Row(color=red) - #+begin_src go -(*pql.Query)(&pql.Query{ - Calls: ([]*pql.Call)([]*pql.Call{ - (*pql.Call)(&pql.Call{ - Name: (string)("Row"), - Args: (map[string]interface{})(map[string]interface{}{ - (string)("color"): (string)("red"), - }), - Children: ([]*pql.Call)(nil), - Type: (pql.CallType)(0), - Precomputed: (map[uint64]interface{})(nil), - }), - }), - callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), - conditional: ([]string)(nil), -}) - #+end_src - #+BEGIN_SRC sqlite - select column from bits where field="color" AND row="red" - #+END_SRC -- [X] Count(Row(color=red)) - #+begin_src go -(*pql.Query)(&pql.Query{ - Calls: ([]*pql.Call)([]*pql.Call{ - (*pql.Call)(&pql.Call{ - Name: (string)("Count"), - Args: (map[string]interface{})(nil), - Children: ([]*pql.Call)([]*pql.Call{ - (*pql.Call)(&pql.Call{ - Name: (string)("Row"), - Args: (map[string]interface{})(map[string]interface{}{ - (string)("color"): (string)("red"), - }), - Children: ([]*pql.Call)(nil), - Type: (pql.CallType)(0), - Precomputed: (map[uint64]interface{})(nil), - }), - }), - Type: (pql.CallType)(0), - Precomputed: (map[uint64]interface{})(nil), - }), - }), - callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), - conditional: ([]string)(nil), -}) - - #+end_src - #+BEGIN_SRC sql - select count(*) from( - select column from bits where field="color" AND row="red" - ) - #+END_SRC -- [X] Intersect(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from bits where field="color" AND row="red" - intersect - select column from bits where field="color" AND row="green" - #+END_SRC -- [X] Union(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from bits where field="color" AND row="red" - union - select column from bits where field="color" AND row="green" - #+END_SRC -- [X] Difference(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from bits where field="color" AND row="red" - except - select column from bits where field="color" AND row="green" - #+END_SRC -- [X] Not(Row(color=red)) - #+BEGIN_SRC sql - select column from columns - except - select column from bits where field="color" AND row="red" - #+END_SRC -- [X] Xor(Row(color=red),Row(color=green)) - #+BEGIN_SRC sql - select column from ( - select column from bits where field="color" AND row="green" - union - select column from bits where field="color" AND row="red" - ) - except - select column from ( - select column from bits where field="color" AND row="green" - intersect - select column from bits where field="color" AND row="red" - ) - #+END_SRC -- [X] Distinct(field=luminosity) - #+BEGIN_SRC sql - select distinct val from bsi where field="lumin" - #+END_SRC -- [X] Distinct(Row(color="red"),field=luminosity) - #+BEGIN_SRC sql - select distinct val from bsi - where field="luminosity" ANDcolumn in ( - select column from bits where row="color" AND row="red" - ) - #+END_SRC -- [X] Distinct(Intersect(Row(color="red"),Row(color="green")),luminosity) - #+BEGIN_SRC sql - select distinct val from bsi - where column in ( - select column from bits where field="color" AND row="red" - intersect - select column from bits where field="color" AND row="green" - ) - AND field="luminosity" - #+END_SRC - -- [X] Count(Union(Intersect(Row(color=red),Row(color=green)),Intersect(Row(color=yellow),Row(color=orange)))) -#+begin_src sqlite -select count(*) from ( -select column from ( - select column from bits where field="color" AND row="green" - intersect - select column from bits where field="color" AND row="red" -) -union -select column from ( - select column from bits where field="color" AND row="yellow" - intersect - select column from bits where field="color" AND row="orange" -)) - -- [ ] Rows(color) -#+begin_src sqlite -select distinct row from bits where field="color" -#+end_src - -- [ ] GroupBy(Rows(color)) - #+BEGIN_SRC sql - select row,count(*) from bits where field="color" group by row; - #+END_SRC -- [ ] GroupBy(Rows(color), Rows(other), limit=7) - #+BEGIN_SRC sql - select row,count(*) from bits where field="color" group by row; - #+END_SRC diff --git a/cmd/sauron/httputil.go b/cmd/sauron/httputil.go deleted file mode 100644 index 7056d40e8..000000000 --- a/cmd/sauron/httputil.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/json" - "log" - "mime" - "net/http" - "strings" - - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/pql" - "github.com/pkg/errors" -) - -// validHeaderAcceptJSON returns false if one or more Accept -// headers are present, but none of them are "application/json" -// (or any matching wildcard). Otherwise returns true. -func validHeaderAcceptJSON(header http.Header) bool { - return validHeaderAcceptType(header, "application", "json") -} - -func validHeaderAcceptType(header http.Header, typ, subtyp string) bool { - if v, found := header["Accept"]; found { - for _, v := range v { - t, _, err := mime.ParseMediaType(v) - if err != nil { - switch err { - case mime.ErrInvalidMediaParameter: - // This is an optional feature, so we can keep going anyway. - default: - continue - } - } - spl := strings.SplitN(t, "/", 2) - if len(spl) < 2 { - continue - } - switch { - case spl[0] == typ && spl[1] == subtyp: - return true - case spl[0] == "*" && spl[1] == subtyp: - return true - case spl[0] == typ && spl[1] == "*": - return true - case spl[0] == "*" && spl[1] == "*": - return true - } - } - return false - } - return true -} - -// successResponse is a general success/error struct for http responses. -type successResponse struct { - //h *Handler - Success bool `json:"success"` - Name string `json:"name,omitempty"` - CreatedAt int64 `json:"createdAt,omitempty"` - //Error *Error `json:"error,omitempty"` - Error error -} - -// check determines success or failure based on the error. -// It also returns the corresponding http status code. -func (r *successResponse) check(err error) (statusCode int) { - if err == nil { - r.Success = true - return 0 - } - - cause := errors.Cause(err) - - // Determine HTTP status code based on the error type. - switch cause.(type) { - case pilosa.BadRequestError: - statusCode = http.StatusBadRequest - case pilosa.ConflictError: - statusCode = http.StatusConflict - case pilosa.NotFoundError: - statusCode = http.StatusNotFound - default: - statusCode = http.StatusInternalServerError - } - - r.Success = false - r.Error = err // = &Error{Message: err.Error()} - - return statusCode -} - -// write sends a response to the http.ResponseWriter based on the success -// status and the error. -func (r *successResponse) write(w http.ResponseWriter, err error) { - // Apply the error and get the status code. - statusCode := r.check(err) - - // Marshal the json response. - msg, err := json.Marshal(r) - if err != nil { - http.Error(w, string(msg), http.StatusInternalServerError) - return - } - - // Write the response. - if statusCode == 0 { - w.Header().Set("Content-Type", "application/json") - _, err := w.Write(msg) - if err != nil { - log.Printf("error writing response: %v", err) - return - } - _, err = w.Write([]byte("\n")) - if err != nil { - log.Printf("error writing newline after response: %v", err) - return - } - } else { - http.Error(w, string(msg), statusCode) - } -} - -type postFieldRequest struct { - Options fieldOptions `json:"options"` -} - -// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values, -// and used for input validation. -type fieldOptions struct { - Type string `json:"type,omitempty"` - CacheType *string `json:"cacheType,omitempty"` - CacheSize *uint32 `json:"cacheSize,omitempty"` - Min *pql.Decimal `json:"min,omitempty"` - Max *pql.Decimal `json:"max,omitempty"` - Scale *int64 `json:"scale,omitempty"` - TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` - Keys *bool `json:"keys,omitempty"` - NoStandardView bool `json:"noStandardView,omitempty"` - ForeignIndex *string `json:"foreignIndex,omitempty"` -} - -func (o *fieldOptions) ToPilosaFieldOptions() *pilosa.FieldOptions { - - fo := &pilosa.FieldOptions{ - Type: o.Type, - - /* - //Base int64 `json:"base,omitempty"` - //BitDepth uint `json:"bitDepth,omitempty"` - Min: *o.Min, - Max: *o.Max, - Scale: *o.Scale, - NoStandardView: o.NoStandardView, - CacheSize: *o.CacheSize, - CacheType: *o.CacheType, - TimeQuantum: *o.TimeQuantum, - ForeignIndex: *o.ForeignIndex, - */ - } - if o.Keys != nil { - fo.Keys = *o.Keys - } - return fo -} - -func (o *fieldOptions) validate() error { - // Pointers to default values. - defaultCacheType := pilosa.DefaultCacheType - defaultCacheSize := uint32(pilosa.DefaultCacheSize) - - switch o.Type { - case pilosa.FieldTypeSet, "": - // Because FieldTypeSet is the default, its arguments are - // not required. Instead, the defaults are applied whenever - // a value does not exist. - if o.Type == "" { - o.Type = pilosa.FieldTypeSet - } - if o.CacheType == nil { - o.CacheType = &defaultCacheType - } - if o.CacheSize == nil { - o.CacheSize = &defaultCacheSize - } - if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type set")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type set")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) - } - case pilosa.FieldTypeInt: - if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) - } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { - return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) - } - case pilosa.FieldTypeDecimal: - if o.Scale == nil { - return pilosa.NewBadRequestError(errors.New("decimal field requires a scale argument")) - } else if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) - } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { - return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) - } - case pilosa.FieldTypeTime: - if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time")) - } else if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type time")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type time")) - } else if o.TimeQuantum == nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time")) - } - case pilosa.FieldTypeMutex: - if o.CacheType == nil { - o.CacheType = &defaultCacheType - } - if o.CacheSize == nil { - o.CacheSize = &defaultCacheSize - } - if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type mutex")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) - } - case pilosa.FieldTypeBool: - if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) - } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) - } else if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) - } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool")) - } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) - } else if o.Keys != nil { - return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool")) - } else if o.ForeignIndex != nil { - return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key")) - } - default: - return errors.Errorf("invalid field type: %s", o.Type) - } - return nil -} diff --git a/cmd/sauron/main.go b/cmd/sauron/main.go deleted file mode 100644 index 497e5346b..000000000 --- a/cmd/sauron/main.go +++ /dev/null @@ -1,1802 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - //"io" - "bytes" - "context" - "database/sql" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "log" - "net" - "net/http" - "os" - "os/exec" - "sort" - "strconv" - "strings" - "time" - - "modernc.org/sqlite" - _ "modernc.org/sqlite" - - //"github.com/gorilla/handlers" - "github.com/gorilla/mux" - "github.com/pilosa/pilosa/v2" - - "github.com/pilosa/pilosa/v2/encoding/proto" - "github.com/pilosa/pilosa/v2/pql" - "github.com/pilosa/pilosa/v2/roaring" - "github.com/pkg/errors" - "github.com/shurcooL/go-goon" -) - -func main() { - -} - -// Sauron sees everything. -// And he converts PQL into SQL. -// -// The PQL is applied to a Pilosa and the SQL -// is applied to a SQL database. Then the -// query results are compared as they are -// returned to the client. -// -type Sauron struct { - cfg *SauronConfig - - sql *PQLToSQL - url string // "https://..." - addr string // host:port - mux *http.ServeMux - srv *http.Server - cmd *exec.Cmd - - // target of proxy - pilosaHttpURL string - - Handler http.Handler -} - -type SauronConfig struct { - Bind string // host:port - BindGRPC string // host:port - GossipPort string // port - DataDir string - - DatabaseName string // for sql database -} - -// IndexInfo2 and FieldInfo2 let us look up -// the schema details quickly by using maps -// to index the fields in an index. -type IndexInfo2 struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options pilosa.IndexOptions `json:"options"` - Fields map[string]*FieldInfo2 `json:"fields"` - ShardWidth uint64 `json:"shardWidth"` -} - -type FieldInfo2 struct { - Name string `json:"name"` - CreatedAt int64 `json:"createdAt,omitempty"` - Options pilosa.FieldOptions `json:"options"` - Views []*pilosa.ViewInfo `json:"views,omitempty"` - indexInfo *IndexInfo2 -} - -func (fi *FieldInfo2) GetIndexName() string { - return fi.indexInfo.Name -} - -func (fi *FieldInfo2) GetIndexCreatedAt() int64 { - return fi.indexInfo.CreatedAt -} - -func (fi *FieldInfo2) GetCreatedAt() int64 { - return fi.CreatedAt -} - -func (fi *FieldInfo2) GetName() string { - return fi.Name -} - -type PQLToSQL struct { - schema pilosa.Schema - DB *sql.DB - - dbprefix string // prefixed to all tables - - // map from indexName to *IndexInfo2 - i2f map[string]*IndexInfo2 - ShardWidth uint64 -} - -func NewPQLToSQL(dbprefix string) *PQLToSQL { - return &PQLToSQL{ - dbprefix: strings.ToLower(dbprefix), // some SQL are not case sensitive, just use lower case. - i2f: make(map[string]*IndexInfo2), - ShardWidth: 1048576, - } -} - -func (ps *PQLToSQL) Start() error { - db, err := sql.Open("sqlite", fmt.Sprintf("%v.db", ps.dbprefix)) - if err != nil { - log.Fatal(err) - } - ps.DB = db - return nil -} -func (ps *PQLToSQL) MustRemove() { - name := fmt.Sprintf("%v.db", ps.dbprefix) - os.Remove(name) - // panicOn(err) -} - -func (ps *PQLToSQL) Stop() error { - return ps.DB.Close() -} -func (ps *PQLToSQL) Store(index, field string, rowIdOrKey interface{}, rowCall []*pql.Call) (bool, error) { - rowSql, _, _, err := ToSql(rowCall, index) - panicOn(err) - insertSql := fmt.Sprintf(`insert into %vλbits select "%v" as field, "%v" as row, column,0 as timestamp from ( %v )`, index, field, rowIdOrKey, rowSql) - res, err := ps.DB.Exec(insertSql) - if err != nil { - return false, err - } - aff, err := res.RowsAffected() - if err != nil { - return false, err - } - return aff > 0, nil -} - -// convert '-' to 'Θ' since db cannot have '-' in table names, -// but pilosa allows it in index and field names. -func dash2theta(s string) (r string) { - return strings.Replace(s, "-", "Θ", -1) -} - -func theta2dash(s string) (r string) { - return strings.Replace(s, "Θ", "-", -1) -} -func (ps *PQLToSQL) CreateSchema(jsonBytes []byte) (err error) { - err = json.Unmarshal(jsonBytes, &ps.schema) - if err != nil { - panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) - } - for _, i := range ps.schema.Indexes { - ps.createDatabase(i.Name) - for _, f := range i.Fields { - panicOn(ps.CreateField(i, f)) - } - } - return nil -} - -func (ps *PQLToSQL) CreateFieldWithOptions(indexName, fieldName string, o fieldOptions) error { - - ii2, ok := ps.i2f[indexName] - if !ok { - return fmt.Errorf("index '%v' not found", indexName) - } - - f2, ok := ii2.Fields[fieldName] - if !ok { - f2 = &FieldInfo2{ - Name: fieldName, - //CreatedAt: f.CreatedAt, - Options: *(o.ToPilosaFieldOptions()), - //Views: f.Views, - indexInfo: ii2, - } - ii2.Fields[fieldName] = f2 - } - return nil -} - -func (ps *PQLToSQL) CreateIndex(indexName string, opts *pilosa.IndexOptions) (err error) { - ii2, ok := ps.i2f[indexName] - if !ok { - ii2 = &IndexInfo2{ - Name: indexName, - //CreatedAt: i.CreatedAt, - Options: *opts, - Fields: make(map[string]*FieldInfo2), - ShardWidth: 1048576, - } - //TODO (twg) figure oiut where to get Shardwidht - ps.i2f[indexName] = ii2 - } - return nil -} -func (ps *PQLToSQL) Query(index string, query []*pql.Call) (err error) { - // sql, err := ToSQL(query, index) - //TODO (twg) implement QUery - return nil -} -func (ps *PQLToSQL) CreateField(i *pilosa.IndexInfo, f *pilosa.FieldInfo) (err error) { - ii2, ok := ps.i2f[i.Name] - if !ok { - ii2 = &IndexInfo2{ - Name: i.Name, - CreatedAt: i.CreatedAt, - Options: i.Options, - Fields: make(map[string]*FieldInfo2), - ShardWidth: i.ShardWidth, - } - ps.i2f[i.Name] = ii2 - } - - f2, ok := ii2.Fields[f.Name] - if !ok { - f2 = &FieldInfo2{ - Name: f.Name, - CreatedAt: f.CreatedAt, - Options: f.Options, - Views: f.Views, - indexInfo: ii2, - } - ii2.Fields[f.Name] = f2 - } - return nil -} - -func (ps *PQLToSQL) createDatabase(index string) (err error) { - idx_prefix := dash2theta(index) + "λ" - // Create fields that don't exist. - sql := fmt.Sprintf("create table %vbits (field string,row,column,timestamp, unique(field,row,column,timestamp) )", idx_prefix) - _, err = ps.DB.Exec(sql) - panicOn(err) - sql = fmt.Sprintf("create table %vbsi (field,column, val bigint,unique(field,column)) ", idx_prefix) - _, err = ps.DB.Exec(sql) - panicOn(err) - sql = fmt.Sprintf(`CREATE VIEW %vcolumns as - select distinct column from %vbits - UNION - select distinct column from %vbsi`, idx_prefix, idx_prefix, idx_prefix) - _, err = ps.DB.Exec(sql) - panicOn(err) - return -} -func (ps *PQLToSQL) GetField(indexName, fieldName string) (*FieldInfo2, error) { - idx, ok := ps.i2f[indexName] - if !ok { - return nil, errors.New(fmt.Sprintf("index not found '%v'", indexName)) - } - f, ok := idx.Fields[fieldName] - if !ok { - return nil, errors.New(fmt.Sprintf("field not found '%v' in index '%v'", fieldName, indexName)) - } - return f, nil -} - -//TODO (twg) below will be moved to its own file later just have it here for quick navigation -type PqlFunction uint8 - -//debate between this -type ResultType uint8 - -const ( - UNKNOWN = ResultType(iota) - BITMAP - INT - ARRAY - GROUPCOUNT - SIGNEDROW - BOOL - PAIRSFIELD - TABLE - AGGSQL -) - -func getAggregateSql(call *pql.Call, index string) (string, error) { - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", errors.New("no _field present") - } - } - var sq string - if len(call.Children) > 0 { - //only child should be a bitmap call - var err error - sq, _, _, err = ToSql(call.Children, index) - if err != nil { - return sq, err - } - sq = " and column in (" + sq + ")" - - } - sqb := fmt.Sprintf(`select %v(val) from %vλbsi where field="%v"%v`, strings.ToLower(call.Name), index, fieldName, sq) - sql := fmt.Sprintf(`select val,count(*) from %vλbsi where val=(%v)%v`, index, sqb, sq) - - return sql, nil -} - -// ToSql takes a pilosa query(ast) and converts it to sql -// assumption: the pql has successfully executed on server -func applyCondition(field string, cond *pql.Condition) (sql string) { - - switch cond.Op { - case pql.EQ, pql.NEQ, pql.LT, pql.LTE, pql.GT, pql.GTE: - val, ok := cond.Uint64Value() - if !ok { - panic("no value for condition") - } - if cond.Op == pql.EQ { - sql = fmt.Sprintf(`field="%v" and val=%d`, field, val) - } else if cond.Op == pql.NEQ { - sql = fmt.Sprintf(`field="%v" and val!=%d`, field, val) - } else if cond.Op == pql.LT { - sql = fmt.Sprintf(`field="%v" and val<%d`, field, val) - } else if cond.Op == pql.LTE { - sql = fmt.Sprintf(`field="%v" and val<=%d`, field, val) - } else if cond.Op == pql.GT { - sql = fmt.Sprintf(`field="%v" and val>%d`, field, val) - } else if cond.Op == pql.GTE { - sql = fmt.Sprintf(`field="%v" and val>=%d`, field, val) - } - case pql.BETWEEN, pql.BTWN_LT_LTE, pql.BTWN_LTE_LT, pql.BTWN_LT_LT: - val, ok := cond.Uint64SliceValue() - if !ok { - panic("bad value for condition") - } - if cond.Op == pql.BETWEEN { - sql = fmt.Sprintf(`field="%v" and %v>=%d and %v<=%d`, field, "val", val[0], "val", val[1]) - } else if cond.Op == pql.BTWN_LT_LTE { - sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[1]) - } else if cond.Op == pql.BTWN_LTE_LT { - sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[0]) - } else if cond.Op == pql.BTWN_LT_LT { - sql = fmt.Sprintf(`field="%v" and %v>%d and %v<%d`, field, "val", val[0], "val", val[1]) - } - } - return -} -func FetchFields(rowsCalls []*pql.Call) ([]string, []interface{}, error) { - //TODO (twg) date fields in rows? - results := make([]string, len(rowsCalls)) - columns := make([]interface{}, 0) - for i, call := range rowsCalls { - //TODO (twg) Time Fields ie from/to args - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return nil, nil, errors.New("no field for rows") - } - } - results[i] = fieldName.(string) - col, ok := call.Args["column"] - if ok { - s := fmt.Sprintf("f%v.column='%v'", i+1, col) - columns = append(columns, s) - } - } - return results, columns, nil -} - -func ToSql(ast []*pql.Call, index string) (string, ResultType, []interface{}, error) { - resultType := UNKNOWN - optional := make([]interface{}, 0) - if len(ast) == 0 { - return "", resultType, optional, nil - } - stack := make([]string, 0) - for _, call := range ast { - switch c := strings.ToLower(call.Name); c { - case "row": - var sql string - if call.HasConditionArg() { - for k, v := range call.Args { - csql := applyCondition(k, v.(*pql.Condition)) - sql = fmt.Sprintf("select distinct column from %vλbsi where %v", index, csql) - - break - } - } else { - field, err := call.FieldArg() - panicOn(err) - row := call.Args[field] - - t, ok := call.Args["from"] - var timestampClause string - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause = fmt.Sprintf(` AND timestamp >= "%v"`, clean) - } - t, ok = call.Args["to"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause += fmt.Sprintf(` AND timestamp < "%v"`, clean) - - } - - sql = fmt.Sprintf(`select distinct column from %vλbits where field="%v" AND row="%v"%v`, index, field, row, timestampClause) - - } - stack = append(stack, sql) - resultType = BITMAP - case "count": - //should have args and should be top level - if len(call.Children) == 0 { - return "", resultType, optional, errors.New("invalid args for count") - } - nestedSQL, _, _, err := ToSql(call.Children, index) - panicOn(err) - sql := "select count(*) from( " + nestedSQL + " )" - stack = append(stack, sql) - resultType = INT - case "union", "intersect": - var sql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(sql) == 0 { - sql = sq - } else { - sql += " " + c + " " + sq - } - } - sql = "select column from(" + sql + ")" - stack = append(stack, sql) - resultType = BITMAP - case "difference": - var sql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(sql) == 0 { - sql = sq - } else { - sql += " except " + sq - } - } - sql = "select column from(" + sql + ")" - stack = append(stack, sql) - resultType = BITMAP - case "not": - //should have args and should be top level - if len(call.Children) == 0 { - return "", resultType, optional, errors.New("invalid args for count") - } - nestedSQL, _, _, err := ToSql(call.Children, index) - panicOn(err) - sql := fmt.Sprintf("select column from %vλcolumns except %v", index, nestedSQL) - stack = append(stack, sql) - resultType = BITMAP - case "xor": - var unionSql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(unionSql) == 0 { - unionSql = sq - } else { - unionSql += " union " + sq - } - } - unionSql = "select column from(" + unionSql + ")" - var intersectSql string - for _, subcall := range call.Children { - sq, _, _, err := ToSql([]*pql.Call{subcall}, index) - panicOn(err) - if len(intersectSql) == 0 { - intersectSql = sq - } else { - intersectSql += " intersect " + sq - } - } - intersectSql = "select column from(" + intersectSql + ")" - sql := unionSql + " except " + intersectSql - stack = append(stack, sql) - resultType = BITMAP - - case "distinct": - //TODO validate field type, only valid for bsi fields - fieldName, ok := call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("field not specified on distinct call") - } - var sql string - if len(call.Children) > 1 { - return "", resultType, optional, errors.New(fmt.Sprintf("unexpected args to distinct query:'%v'", call.Children)) - } - sql = fmt.Sprintf(`select distinct val from %vλbsi where field="%v"`, index, fieldName) - if len(call.Children) > 0 { - sq, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) - panicOn(err) - sql += " AND column in (" + sq + ")" - } - stack = append(stack, sql) - resultType = SIGNEDROW - case "rows": - //TODO (twg) Time Fields ie from/to args - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("no _field present") - } - } - optional = append(optional, fieldName) - sql := fmt.Sprintf(`select distinct field, row from %vλbits where field="%v"`, index, fieldName) - col, ok := call.Args["column"] - if ok { - sql = fmt.Sprintf("%v and column='%v'", sql, col) - } - prev, ok := call.Args["previous"] - if ok { - sql = fmt.Sprintf("%v and row>'%v'", sql, prev) - } - t, ok := call.Args["from"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) - sql += timestampClause - } - t, ok = call.Args["to"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) - sql += timestampClause - } - sql += " order by row" - limit, ok := call.Args["limit"] - if ok { - sql = fmt.Sprintf("%v limit %v", sql, limit) - } - stack = append(stack, sql) - resultType = ARRAY - case "groupby": - //need to figure out how many fields we will be looking at - fields, optColumns, err := FetchFields(call.Children) - panicOn(err) - sql := "select " - for i := range fields { - f := fmt.Sprintf("f%d", i+1) - if i == 0 { - sql += fmt.Sprintf(" %s.field, %s.row", f, f) - } else { - - sql += fmt.Sprintf(",%s.field, %s.row", f, f) - } - } - tname := fmt.Sprintf("%vλbits", index) - sql += fmt.Sprintf(",count(*) as cnt from %v f1", tname) - if len(fields) > 1 { - for i := range fields[1:] { - f := fmt.Sprintf("f%d", i+2) - sql += fmt.Sprintf(" inner join %v %v on f1.column = %v.column", tname, f, f) - } - } - sql += " where " - gb := "" - for i, field := range fields { - f := fmt.Sprintf("f%d", i+1) - if i == 0 { - sql += fmt.Sprintf("%v.field='%v'", f, field) - gb += fmt.Sprintf("%v.field,%v.row", f, f) - } else { - sql += fmt.Sprintf("and %v.field='%v'", f, field) - gb += fmt.Sprintf(",%v.field,%v.row", f, f) - } - } - if len(optColumns) > 0 { - for _, column := range optColumns { - sql += fmt.Sprintf(" and %v ", column) - } - } - sql += " group by " + gb + " order by cnt desc" - - optional = append(optional, len(fields)) - - stack = append(stack, sql) - resultType = GROUPCOUNT - case "topn": - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("no _field present") - } - } - var sq string - if len(call.Children) > 0 { - if len(call.Children) > 1 { - return "", resultType, optional, errors.New("invalid topn") - } - var err error - sq, _, _, err = ToSql(call.Children, index) - panicOn(err) - sq = " and column in (" + sq + ")" - - } - sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) - - limit, ok := call.Args["n"] - if ok { - sql += fmt.Sprintf(" limit %v", limit) - } - stack = append(stack, sql) - resultType = PAIRSFIELD - case "topk": - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - return "", resultType, optional, errors.New("no _field present") - } - } - var sq string - filter, ok := call.Args["filter"] - if ok { - //assuming this works in pilosa so the filter must be a correct bitmap call - //ie row/union/intersect/xor/difference - var err error - sq, _, _, err = ToSql([]*pql.Call{filter.(*pql.Call)}, index) - panicOn(err) - sq = " and column in (" + sq + ")" - - } - t, ok := call.Args["from"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) - sq += timestampClause - } - t, ok = call.Args["to"] - if ok { - clean := strings.Replace(t.(string), "T", " ", 1) - timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) - sq += timestampClause - } - sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) - - limit, ok := call.Args["k"] - if ok { - sql += fmt.Sprintf(" limit %v", limit) - } - stack = append(stack, sql) - resultType = PAIRSFIELD - case "min", "max": - sql, err := getAggregateSql(call, index) - panicOn(err) - stack = append(stack, sql) - resultType = AGGSQL - case "sum": - fieldName, ok := call.Args["_field"] - if !ok { - fieldName, ok = call.Args["field"] - if !ok { - panic("no _field present") - } - } - var sq string - if len(call.Children) > 0 { - //only child should be a bitmap call - var err error - sq, _, _, err = ToSql(call.Children, index) - panicOn(err) - sq = " and column in (" + sq + ")" - - } - sql := fmt.Sprintf(`select sum(val),count(*) from %vλbsi where field="%v"%v`, index, fieldName, sq) - stack = append(stack, sql) - resultType = AGGSQL - case "constrow": - sql := "select column1 as column from (values " - columns, ok := call.Args["columns"].([]interface{}) - - if ok { - for i, col := range columns { - if i != 0 { - sql += "," - } - sql += fmt.Sprintf("(%v)", col) - - } - sql += ")" - - } - - stack = append(stack, sql) - resultType = BITMAP - case "all": - sql := fmt.Sprintf("select column from %vλcolumns", index) - stack = append(stack, sql) - resultType = BITMAP - case "extract": - // extract is a table response - bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) - panicOn(err) - var rowsPart string - if len(call.Children) > 1 { - rows := call.Children[1:] - rowsPart = "select field,row from(" - for i, row := range rows { - part, _, _, err1 := ToSql([]*pql.Call{row}, index) - panicOn(err1) - if i > 0 { - rowsPart += "union all select field,row from (" - } - rowsPart += part + ")" - } - } - var sql string - if len(rowsPart) == 0 { - sql = "select '' as field, '' as row,column from (" + bm + ")" - } else { - sql = "select sq.field, sq.row, b.column from (" - sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) - sql += " where b.column in (" + bm + ")" - } - - stack = append(stack, sql) - resultType = TABLE - case "unionrows": - var rowsPart string - rowsPart = "select field,row from(" - for i, row := range call.Children { - part, _, _, err1 := ToSql([]*pql.Call{row}, index) - panicOn(err1) - if i > 0 { - rowsPart += "union all select field,row from (" - } - rowsPart += part + ")" - } - sql := "select distinct b.column from (" - sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) - stack = append(stack, sql) - resultType = BITMAP - case "includescolumn": - bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) - panicOn(err) - col, ok := call.Args["column"] - if !ok { - return "", resultType, optional, errors.New("missing column parameter") - } - - sql := fmt.Sprintf("select count(*)>0 from (select column from(%v) where column='%v')", bm, col) - stack = append(stack, sql) - resultType = BOOL - default: - goon.Dump(call) - return "", resultType, optional, errors.New(fmt.Sprintf("unknown call '%v'", c)) - } - } - if len(stack) == 0 { - //not sure if i should error on an NOP - return "", resultType, optional, nil - } - return stack[0], resultType, optional, nil -} - -func NewSauron(cfg *SauronConfig) *Sauron { - - addr := "127.0.0.1:8080" - url := "http://" + addr - s := &Sauron{ - cfg: cfg, - addr: addr, - url: url, - sql: NewPQLToSQL(cfg.DatabaseName), - srv: &http.Server{ - Addr: addr, - ReadTimeout: 50 * time.Millisecond, - WriteTimeout: 10 * time.Second, - MaxHeaderBytes: 1 << 20, - }, - } - s.Handler = newRouter(s) - s.srv.Handler = s.Handler - return s -} - -func (p *Sauron) Start() (urlstring string, err error) { - started := make(chan struct{}) - - panicOn(p.sql.Start()) - - // start a pilosa server - bind := p.cfg.Bind - dataDir := p.cfg.DataDir - bindGRPC := p.cfg.BindGRPC - gossipPort := p.cfg.GossipPort - - p.cmd = exec.Command("pilosa", "server", - "--bind", bind, - "--data-dir", dataDir, - "--bind-grpc", bindGRPC, - "--gossip.port", gossipPort) - err = p.cmd.Start() - panicOn(err) - - // wait for it to be listening - p.pilosaHttpURL = "http://" + bind - ok := false - seconds := 20 - for i := 0; i < seconds; i++ { - nc, err := net.Dial("tcp", bind) - if err == nil { - nc.Close() - ok = true - break - } - time.Sleep(time.Second) - } - if !ok { - return "", fmt.Errorf("could not contact pilosa server on %v after %v seconds. pid = %v", bind, seconds, p.cmd.Process.Pid) - } - go func() { - close(started) - - err = p.srv.ListenAndServe() - if err != nil { - if strings.Contains(err.Error(), "Server closed") { - return - } - } - panicOn(err) - }() - <-started - return p.url, nil -} - -func (p *Sauron) Stop() { - panicOn(p.sql.Stop()) - err := p.srv.Shutdown(context.Background()) - panicOn(err) - // Kill pilosa - if err = p.cmd.Process.Kill(); err != nil { - vv("failed to kill process: ", err) - } -} - -// GetAvailPort asks the OS for an unused port. -// There's a race here, where the port could be grabbed by someone else -// before the caller gets to Listen on it, but we are only using -// it to find a random port for the test hang debugging. -// Moreover, in practice such races are rare. Just ask for -// it again if the port is taken. -// Uses net.Listen("tcp", ":0") to determine a free port, then -// releases it back to the OS with Listener.Close(). -func GetAvailPort() int { - l, _ := net.Listen("tcp", ":0") - r := l.Addr() - l.Close() - return r.(*net.TCPAddr).Port -} - -// newRouter creates a new mux http router. -func newRouter(handler *Sauron) http.Handler { - router := mux.NewRouter() - - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") - router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") - router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") - - router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex") - router.HandleFunc("/index/{index}/field/", handler.handlePostField).Methods("POST").Name("PostField") - router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField") - router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring") - router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport") - - return router -} - -// handleGetSchema handles GET /schema requests. -func (s *Sauron) handleGetSchema(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - _ = body - _ = resp - _ = err - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(resp.StatusCode) - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - _, err = w.Write(respBody) - panicOn(err) - - // B) there is no SQL version of get schema. - -} - -// creates an entire schema from the dumped schema. -// requires completed structures, no defaults applied -func (s *Sauron) handlePostSchema(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - - w.WriteHeader(resp.StatusCode) - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - _, err = w.Write(respBody) - panicOn(err) - - // B) SQL - panicOn(s.sql.CreateSchema(body)) -} - -// handlePostField handles /index/{index}/field/{field} requests -func (s *Sauron) handlePostIndex(w http.ResponseWriter, r *http.Request) { - body, resp, err := s.ForwardHandler(w, r) - _ = body - _ = resp - _ = err - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - indexName, ok := mux.Vars(r)["index"] - if !ok { - http.Error(w, "index name is required", http.StatusBadRequest) - return - } - req := struct { - Options pilosa.IndexOptions - }{ - Options: pilosa.IndexOptions{ - Keys: false, - TrackExistence: true, - }} - if len(body) > 0 { //only process if options are present - err = json.Unmarshal(body, &req) - panicOn(err) - } - err = s.sql.CreateIndex(indexName, &req.Options) - panicOn(err) - w.Write(respBody) -} - -// handlePostField handles /index/{index}/field/{field} requests -func (s *Sauron) handlePostField(w http.ResponseWriter, r *http.Request) { - body, resp, err := s.ForwardHandler(w, r) - _ = body - _ = resp - _ = err - indexName, ok := mux.Vars(r)["index"] - if !ok { - http.Error(w, "index name is required", http.StatusBadRequest) - return - } - - fieldName, ok := mux.Vars(r)["field"] - if !ok { - http.Error(w, "field name is required", http.StatusBadRequest) - return - } - - respBody, err := ioutil.ReadAll(resp.Body) - panicOn(err) - // Decode request. - var req postFieldRequest - dec := json.NewDecoder(bytes.NewReader(body)) - dec.DisallowUnknownFields() - err = dec.Decode(&req) - if err != io.EOF { - panicOn(err) - } - err = s.sql.CreateFieldWithOptions(indexName, fieldName, req.Options) - _, err = w.Write(respBody) - panicOn(err) -} - -// handlePostQuery handles /index/{index}/query requests -func (s *Sauron) handlePostQuery(w http.ResponseWriter, req *http.Request) { - - body, resp, err := s.ForwardHandler(w, req) - pilosaJSON, err := ioutil.ReadAll(resp.Body) - panicOn(err) - var pilosaResponse pilosa.QueryResponse // DEBUG - dec := json.NewDecoder(bytes.NewReader(pilosaJSON)) - err = dec.Decode(&pilosaResponse) - panicOn(err) - qreq := &pilosa.QueryRequest{ - Index: mux.Vars(req)["index"], - Query: string(body), - } - - oracleResponse, err := s.sql.IssueQuery(qreq) - - if !pilosaResponse.Equals(oracleResponse) { - //plan is to write out a useful query response indicating a discrepancy - //TODO (twg) write diff in response somehow - vv("oracle:") - prettyJson(*oracleResponse) - vv("pilosa:") - prettyJson(pilosaResponse) - panic("barf") - } -} -func prettyJson(a pilosa.QueryResponse) { - src, _ := a.MarshalJSON() - dst := &bytes.Buffer{} - if err := json.Indent(dst, src, "", " "); err != nil { - panic(err) - } - vv("%v", string(dst.Bytes())) -} - -func (s *Sauron) FieldInfo(index, field string) (*FieldInfo2, error) { - idx, ok := s.sql.i2f[index] - if ok { - f, ok := idx.Fields[field] - if ok { - return f, nil - } - return nil, errors.New(fmt.Sprintf("field not in schema '%v'", field)) - } - return nil, errors.New(fmt.Sprintf("index not in schema '%v'", index)) -} -func (s *Sauron) handlePostImport(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - _ = body - _ = resp - _ = err - // Get index and field type to determine how to handle the - // import data. - indexName := mux.Vars(req)["index"] - fieldName := mux.Vars(req)["field"] - - _ = indexName - _ = fieldName - - // If the clear flag is true, treat the import as clear bits. - q := req.URL.Query() - doClear := q.Get("clear") == "true" - doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" - _ = doClear - _ = doIgnoreKeyCheck - - info, err := s.FieldInfo(indexName, fieldName) - panicOn(err) - // Unmarshal request based on field type. - fieldType := info.Options.Type - if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal { - iv := &pilosa.ImportValueRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, iv); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - //TODO (twg) ignore columnKeys at the moment - //do importValue - for i, column := range iv.ColumnIDs { - if len(iv.Values) > 0 { - s.sql.Set(indexName, column, fieldName, iv.Values[i], "") - } else if len(iv.FloatValues) > 0 { //TODO (twg) these may not be reachable - s.sql.Set(indexName, column, fieldName, iv.FloatValues[i], "") //probably some issues here - } else if len(iv.StringValues) > 0 { - s.sql.Set(indexName, column, fieldName, iv.StringValues[i], "") - } - } - } else { - // Field type: set, time, mutex - // Marshal into request object. - ir := &pilosa.ImportRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, ir); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - //do import - action := s.sql.Set - if ir.Clear { - action = func(index string, column uint64, field string, value interface{}, ts string) (bool, error) { - return s.sql.Clear(index, column, field, value) - } - } - changed := false - _ = changed - for i, column := range ir.ColumnIDs { - ts := "" - if len(ir.Timestamps) > 0 { - t := time.Unix(ir.Timestamps[i], 0) - ts = t.Format("2006-01-02 15:04:05") - } - if len(ir.RowIDs) > 0 { - change, err := action(indexName, column, fieldName, ir.RowIDs[i], ts) - panicOn(err) - if change { - changed = true - } - } else if len(ir.RowKeys) > 0 { //TODO (twg) these may not be reachable - change, err := action(indexName, column, fieldName, ir.RowKeys[i], ts) - panicOn(err) - if change { - changed = true - } - } - } - } -} -func (s *Sauron) handlePostImportRoaring(w http.ResponseWriter, req *http.Request) { - body, resp, err := s.ForwardHandler(w, req) - _ = body - _ = resp - _ = err - // Get index and field type to determine how to handle the - // import data. - indexName := mux.Vars(req)["index"] - fieldName := mux.Vars(req)["field"] - - irr := &pilosa.ImportRoaringRequest{} - err = proto.DefaultSerializer.Unmarshal(body, irr) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - //do i need the shard? - urlVars := mux.Vars(req) - shard, err := strconv.ParseUint(urlVars["shard"], 10, 64) - if err != nil { - panicOn(err) - } - for v, blob := range irr.Views { - _ = v //ignore the view for now - rit, err := roaring.NewRoaringIterator(blob) - panicOn(err) - s.sql.ImportRoaring(indexName, fieldName, shard, rit) - } -} - -//end handler -func (s *Sauron) ForwardHandler(w http.ResponseWriter, req *http.Request) (body []byte, resp *http.Response, err error) { - // TODO (twg) replace all the instances - // we need to buffer the body if we want to read it here and send it - // in the request. - body, err = ioutil.ReadAll(req.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // you can reassign the body if you need to parse it as multipart - req.Body = ioutil.NopCloser(bytes.NewReader(body)) - - // create a new url from the raw RequestURI sent by the client - url := fmt.Sprintf("%s%s", s.pilosaHttpURL, req.RequestURI) - - proxyReq, err := http.NewRequest(req.Method, url, bytes.NewReader(body)) - - // We may want to filter some headers, otherwise we could just use a shallow copy - // proxyReq.Header = req.Header - proxyReq.Header = make(http.Header) - for h, val := range req.Header { - proxyReq.Header[h] = val - } - - resp, err = http.DefaultClient.Do(proxyReq) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return - } - return -} - -// ImportRoaring must not be a bsi field..check higher up the stack -func (ps *PQLToSQL) ImportRoaring(index, field string, shard uint64, itr roaring.RoaringIterator) { - rb := roaring.NewBitmap() - for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { - rb.Put(itrKey, synthC) - } - for _, pos := range rb.Slice() { - row, column := pos/ps.ShardWidth, (shard*ps.ShardWidth)+(pos%ps.ShardWidth) - //ps.SetWithInt64Field(index, column, field, int64(row)) - ps.Set(index, column, field, int64(row), "") - } -} - -func (ps *PQLToSQL) isBSI(index, field string) (full string, isBSI bool) { - idx, ok := ps.i2f[index] - if !ok { - panic(fmt.Errorf("isBSI: no such index '%v'", index)) - } - - fld, ok := idx.Fields[field] - if !ok { - panic(fmt.Errorf("isBSI: no such field '%v'", field)) - } - if fld.Options.Type == "int" { - return dash2theta(index) + "λbsi", true - } - return dash2theta(index) + "λbits", false -} - -// Set is the basic bit operation and value operation as well, depending on the field type -func (ps *PQLToSQL) Set(index string, column uint64, field string, value interface{}, timestamp string) (bool, error) { - table, isBSI := ps.isBSI(index, field) - var sql string - if isBSI { - sql = fmt.Sprintf("INSERT INTO %v VALUES( '%v','%v', '%v')", table, field, column, value) - } else { - //if len(timestamp) == 0 { - // sql = fmt.Sprintf("INSERT INTO %v (field,row,column)VALUES( '%v','%v', '%v')", table, field, value, column) - // } else { - //TODO(twg) will need to make the timestamp as granular as the field is configured. - sql = fmt.Sprintf("INSERT INTO %v (field,row,column,timestamp)VALUES( '%v','%v', '%v','%v')", table, field, value, column, timestamp) - // } - } - - _, err := ps.DB.Exec(sql) - if err != nil { - if sqerr, ok := err.(*sqlite.Error); ok { - if sqerr.Code() == 2067 { //constraint error - return false, nil - } - } - } - panicOn(err) - return true, nil -} - -// Clear removes the specified column or value -func (ps *PQLToSQL) Clear(index string, column uint64, field string, value interface{}) (bool, error) { - table, isBSI := ps.isBSI(index, field) - var sql string - if isBSI { - sql = fmt.Sprintf("delete from %v where field='%v' and column='%v'", table, field, column) - } else { - sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' and column='%v'", table, field, value, column) - } - - res, err := ps.DB.Exec(sql) - panicOn(err) - r, err := res.RowsAffected() - panicOn(err) - if r == 0 { - return false, nil - } - return true, nil -} - -// ClearRow removes entire Row(feature) -func (ps *PQLToSQL) ClearRow(index string, field string, row interface{}) (bool, error) { - table, isBSI := ps.isBSI(index, field) - var sql string - if isBSI { - panic("bsi can't clear row allowed") - } else { - sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' ", table, field, row) - } - - r, err := ps.DB.Exec(sql) - panicOn(err) - n, err := r.RowsAffected() - return n > 0, err -} - -func sliceToString(slc []int64) (r string) { - if len(slc) == 0 { - return - } - for _, i := range slc { - r += fmt.Sprintf("%v,", i) - } - return r[:len(r)-1] -} - -const columnLabel = "col" - -func (ps *PQLToSQL) IssueQuery(req *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { - index := req.Index - q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() - if err != nil { - return nil, errors.Wrap(err, "PQLToSQL.IssueQuery parsing PQL") - } - var results pilosa.QueryResponse - for _, c := range q.Calls { - - if callIndex := c.CallIndex(); callIndex != "" { - index = callIndex - } - - // Handle the field arg. - switch strings.ToLower(c.Name) { - //handle the write queries - //TODO (twg) figure out sqlite changes() function to determine number of bits changed - case "set": - // Read colID. - colID, ok, err := c.UintArg("_" + columnLabel) - if err != nil { - return nil, fmt.Errorf("reading Set() column: %v", err) - } else if !ok { - return nil, fmt.Errorf("Set() column argument '%v' required", columnLabel) - } - ts, ok := c.Args["_timestamp"] - var timestamp string - if ok { - timestamp = strings.Replace(ts.(string), "T", " ", 1) - } - - field, err := c.FieldArg() - if err == nil { - rs, err := ps.Set(index, colID, field, c.Args[field], timestamp) - panicOn(err) - results.Results = append(results.Results, rs) - } else { - return nil, err - } - case "clear": - // Read colID. - colID, ok, err := c.UintArg("_" + columnLabel) - if err != nil { - return nil, fmt.Errorf("reading Clear() column: %v", err) - } else if !ok { - return nil, fmt.Errorf("Clear() column argument '%v' required", columnLabel) - } - - field, err := c.FieldArg() - if err == nil { - rs, err := ps.Clear(index, colID, field, c.Args[field]) - if err != nil { - return nil, fmt.Errorf("Clear() column: %v", err) - } - results.Results = append(results.Results, rs) - } else { - vv("c.FieldArg() returned '%v'", err) - } - case "clearrow": - field, err := c.FieldArg() - panicOn(err) - rs, err := ps.ClearRow(index, field, c.Args[field]) - panicOn(err) - results.Results = append(results.Results, rs) - case "store": - field, err := c.FieldArg() - panicOn(err) - rs, err := ps.Store(index, field, c.Args[field], c.Children) - panicOn(err) - results.Results = append(results.Results, rs) - default: //TODO (twg) move ToSQL into main package - rs, err := ps.readQuery(index, q) - panicOn(err) - results.Results = append(results.Results, rs) - } - } - return &results, nil -} -func (ps *PQLToSQL) handleBitmapCall(sql string) (*pilosa.Row, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - defer rows.Close() - columns := make([]uint64, 0) - for rows.Next() { - //really need a way to know what the results look like - var column uint64 - rows.Scan(&column) - columns = append(columns, column) - } - row := pilosa.NewRow(columns...) - return row, nil -} - -func (ps *PQLToSQL) handleIntCall(sql string) (count uint64, err error) { - rows, err := ps.DB.Query(sql) - if err != nil { - return 0, err - } - defer rows.Close() - for rows.Next() { - //really need a way to know what the results look like - rows.Scan(&count) - return - } - return -} - -type Table struct { -} - -func (ps *PQLToSQL) handleSignedRowCall(sql string) (pilosa.SignedRow, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - - var id uint64 - var pos []uint64 - var neg []uint64 - defer rows.Close() - for rows.Next() { - err := rows.Scan(&id) - if err != nil { - return pilosa.SignedRow{}, err - } - if id >= 0 { - pos = append(pos, id) - } else { - neg = append(neg, id) - } - } - err = rows.Err() - if err != nil { - return pilosa.SignedRow{}, err - } - posRow := pilosa.NewRow(pos...) - negRow := pilosa.NewRow(neg...) - return pilosa.SignedRow{Pos: posRow, Neg: negRow}, nil - -} -func (ps *PQLToSQL) handleStringArrayCall(sql string) ([]interface{}, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - rowKeys := make([]interface{}, 0) - var field string - var id string - defer rows.Close() - for rows.Next() { - err := rows.Scan(&field, &id) - if err != nil { - return nil, err - } - rowKeys = append(rowKeys, id) - } - err = rows.Err() - if err != nil { - return nil, err - } - return rowKeys, nil -} - -func (ps *PQLToSQL) handleIntArrayCall(sql string) (map[string][]interface{}, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - rowIDs := make([]interface{}, 0) - var id uint64 - var field string - defer rows.Close() - for rows.Next() { - err := rows.Scan(&field, &id) - if err != nil { - return nil, err - } - rowIDs = append(rowIDs, id) - } - err = rows.Err() - if err != nil { - return nil, err - } - m := map[string][]interface{}{"rows": rowIDs, "keys": []interface{}{}} - return m, nil -} -func (ps *PQLToSQL) handleGroupCountCall(sqlin string, index string, numFields int) (pilosa.SortedGroupCount, error) { - rows, err := ps.DB.Query(sqlin) - panicOn(err) - //groupCounts := make([]pilosa.GroupCount, 0) - var groupCounts pilosa.SortedGroupCount - defer rows.Close() - cols, err := rows.Columns() - if err != nil { - return nil, err - } - row := make([]interface{}, len(cols)) - for i := range cols { - row[i] = new(sql.RawBytes) - } - for rows.Next() { - // need a row*NumFields + cnt - //err := rows.Scan(&field, &row, &count) - groupCount := pilosa.GroupCount{} - - err := rows.Scan(row...) - if err != nil { - return nil, err - } - for i := 0; i < len(row)-1; i += 2 { - ptr := row[i].(*sql.RawBytes) - fieldName := string(*ptr) - fr := pilosa.FieldRow{Field: fieldName} - fi, err := ps.GetField(index, fieldName) - if err != nil { - return nil, err - } - ptr = row[i+1].(*sql.RawBytes) - v := string(*ptr) - if fi.Options.Keys { - fr.RowKey = v - - } else { - iv, err := strconv.Atoi(v) - if err != nil { - return nil, err - } - fr.RowID = uint64(iv) - } - - groupCount.Group = append(groupCount.Group, fr) - - } - ptr := row[len(row)-1].(*sql.RawBytes) - data := string(*ptr) //convert to string - i, err := strconv.ParseInt(data, 10, 64) - panicOn(err) - groupCount.Count = uint64(i) - groupCounts = append(groupCounts, groupCount) - - } - - err = rows.Err() - if err != nil { - return nil, err - } - return groupCounts, nil -} -func (ps *PQLToSQL) handlePairsFieldCall(sql, index string) (pilosa.PairsField, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - var count uint64 - var row string - defer rows.Close() - res := pilosa.PairsField{} - for rows.Next() { - err := rows.Scan(&row, &count) - if err != nil { - return pilosa.PairsField{}, err - } - pair := pilosa.Pair{Key: row, Count: count} - res.Pairs = append(res.Pairs, pair) - } - err = rows.Err() - if err != nil { - return pilosa.PairsField{}, err - } - return res, nil -} - -// special structures to deal with Extract. In the future I would like to use -// pilosa structs, but dealing with keys makes that complicated for now -// note the column is a string, but i only worry about the type when marshalling out for -// the sqlite part is typeless. Handling keys is almost seemless and not requiring translation -type extractColumns struct { - column string - rows map[int][]interface{} - parent *sauronExtractedIDMatrix -} - -func (eid extractColumns) MarshalJSON() ([]byte, error) { - buf := new(bytes.Buffer) - buf.WriteString(fmt.Sprintf(`{"column": %v,"rows":[`, eid.column)) - rows := make([][]interface{}, len(eid.parent.fields)) - for k, v := range eid.rows { - rows[k] = v - } - for i, grow := range rows { - if i != 0 { - buf.WriteByte(44) //write the comma - } - isKey := eid.parent.fields[i].isKey - buf.WriteString(`[`) - for d, row := range grow { - if d != 0 { - buf.WriteByte(44) //write the comma - } - if isKey { - buf.WriteString(fmt.Sprintf(`"%v"`, row)) - } else { - buf.WriteString(fmt.Sprintf(`%v`, row)) - } - - } - buf.WriteString(`]`) - } - - buf.WriteString(`]}`) - return buf.Bytes(), nil -} - -type extractFields struct { - Name string `json:"name"` - Type string `json:"type"` - isKey bool -} -type sauronExtractedIDMatrix struct { - columns []extractColumns - fields []extractFields -} - -func (eid sauronExtractedIDMatrix) MarshalJSON() ([]byte, error) { - buf := new(bytes.Buffer) - sort.Slice(eid.columns, func(i, j int) bool { - return eid.columns[i].column < eid.columns[j].column - }) - buf.WriteString(`{"columns":[`) - for i, col := range eid.columns { - b, err := col.MarshalJSON() - panicOn(err) - if i != 0 { - buf.WriteByte(44) //write the comma - } - buf.Write(b) - } - - buf.WriteString(`],"fields":[`) - for i, f := range eid.fields { - if i != 0 { - buf.WriteByte(44) - } - b, err := json.Marshal(f) - panicOn(err) - buf.Write(b) - } - buf.WriteString(`]}`) - return buf.Bytes(), nil -} - -func (ps *PQLToSQL) handleTableCall(sql, index string) (sauronExtractedIDMatrix, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - var field string - var row string - var column string - defer rows.Close() - col := make(map[string]map[string][]interface{}) - fidx := make(map[string]int) - results := sauronExtractedIDMatrix{} - counter := 0 - empty := make(map[string][]interface{}) - for rows.Next() { - err := rows.Scan(&field, &row, &column) - if err != nil { - return sauronExtractedIDMatrix{}, err - } - if field != "" { - _, ok := fidx[field] - if !ok { - fidx[field] = counter - counter++ - } - rows, ok := col[column] - if !ok { - rows = make(map[string][]interface{}) - } - rows[field] = append(rows[field], row) - //results.Columns = append(results.Columns, pilosa.ExtractedIDColumn{ColumnID: column, Rows: [][]uint64{}}) - col[column] = rows - } else { //constrow, or no filed info - col[column] = empty - } - } - err = rows.Err() - if err != nil { - return sauronExtractedIDMatrix{}, err - } - for c, rowMap := range col { - idc := extractColumns{rows: make(map[int][]interface{}), parent: &results} - idc.column = c - for fld, rows := range rowMap { - idx := fidx[fld] - idc.rows[idx] = rows - } - results.columns = append(results.columns, idc) - } - results.fields = make([]extractFields, len(fidx)) - for fieldName, idx := range fidx { - fi, _ := ps.GetField(index, fieldName) - // TODO (twg) hack need to figure out proper way to determine type - aType := "[]uint64" - if fi.Options.Keys { - aType = "[]string" - } - results.fields[idx] = extractFields{Name: fieldName, Type: aType, isKey: fi.Options.Keys} - } - - return results, nil -} - -func (ps *PQLToSQL) handleAggSQLCall(sql, index string) (pilosa.ValCount, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - defer rows.Close() - results := pilosa.ValCount{} - var val int64 - var count int64 - for rows.Next() { - err := rows.Scan(&val, &count) - if err != nil { - return pilosa.ValCount{}, err - } - results.Count = count - results.Val = val - break //shouldn't be anymore rows but being safe - } - err = rows.Err() - if err != nil { - return pilosa.ValCount{}, err - } - return results, nil -} -func (ps *PQLToSQL) handleBoolCall(sql, index string) (bool, error) { - rows, err := ps.DB.Query(sql) - panicOn(err) - defer rows.Close() - var val bool - for rows.Next() { - err := rows.Scan(&val) - if err != nil { - return false, err - } - break //shouldn't be anymore rows but being safe - } - err = rows.Err() - if err != nil { - return val, err - } - return val, nil -} - -func (ps *PQLToSQL) readQuery(index string, query *pql.Query) (interface{}, error) { - //Row(color=red) - sql, resultType, optional, err := ToSql(query.Calls, index) - panicOn(err) - switch resultType { - case BITMAP: - return ps.handleBitmapCall(sql) - case INT: - return ps.handleIntCall(sql) - case SIGNEDROW: - return ps.handleSignedRowCall(sql) - case ARRAY: - fieldName := optional[0].(string) - info, ok := ps.i2f[index] - if !ok { - return nil, errors.New(fmt.Sprintf("index not in schema %v", index)) - } - field, ok := info.Fields[fieldName] - if !ok { - return nil, errors.New(fmt.Sprintf("field not in schema %v", fieldName)) - } - if field.Options.Keys { - return ps.handleStringArrayCall(sql) - } else { - return ps.handleIntArrayCall(sql) - } - case GROUPCOUNT: - return ps.handleGroupCountCall(sql, index, optional[0].(int)) - case PAIRSFIELD: - return ps.handlePairsFieldCall(sql, index) - case TABLE: - return ps.handleTableCall(sql, index) - case AGGSQL: - return ps.handleAggSQLCall(sql, index) - case BOOL: - return ps.handleBoolCall(sql, index) - default: - vv("Unknown query Type %v", resultType) - } - return nil, nil -} - -type Int64Slice []int64 - -func (p Int64Slice) Len() int { return len(p) } -func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } -func (p Int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - -// on tables starting with ps.dbprefix will be returned. -func (ps *PQLToSQL) ListIndexFields() (tables []string) { - for indexName, index := range ps.i2f { - for fieldname, _ := range index.Fields { - - tables = append(tables, indexName+"/"+fieldname) - } - } - sort.Strings(tables) - return tables -} diff --git a/cmd/sauron/populate.sh b/cmd/sauron/populate.sh deleted file mode 100644 index 46e7dae67..000000000 --- a/cmd/sauron/populate.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -mod=20 -for i in `seq 100 1`;do -b=$(dc -e "$i $mod %p") -[[ $b == 0 ]] && mod=$(($mod+1)) -r=$(($b+30)) -payload="Set($i, j=$r)" -echo $payload -curl -XPOST "localhost:10101/index/i/query" -d "$payload" -done diff --git a/cmd/sauron/sample_schema.json b/cmd/sauron/sample_schema.json deleted file mode 100644 index 4dcc2c501..000000000 --- a/cmd/sauron/sample_schema.json +++ /dev/null @@ -1 +0,0 @@ -{"indexes":[{"name":"trait_store","createdAt":1595896639330112903,"options":{"keys":true,"trackExistence":true},"fields":[{"name":"aba","createdAt":1595896641512346986,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"ach_batch_recency","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_pass_thru_recency","createdAt":1595896640333428009,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_payment_recency","createdAt":1595896642230184052,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"bools","createdAt":1595896640151751600,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"bools-exists","createdAt":1595896640855732760,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"central_group","createdAt":1595896640036032355,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"custom_audiences","createdAt":1595896643284443107,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"days_since_last_logon","createdAt":1595896639332677618,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":1,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"db","createdAt":1595896639548206413,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"desktop_recency","createdAt":1595896642633713852,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"domestic_wire_recency","createdAt":1595896642264402807,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"external_transfer_recency","createdAt":1595896639332854139,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"fields","createdAt":1595896639552562813,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"funds_transfer_recency","createdAt":1595896640803192694,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"international_wire_recency","createdAt":1595896643511015744,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"mobile_remote_deposit_recency","createdAt":1595896641174868205,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"payroll_recency","createdAt":1595896642880416952,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"pfm_category_total_current_balance__401k_investment","createdAt":1595896641492935339,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__403b_investment","createdAt":1595896639553865549,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__529_investment","createdAt":1595896643478533291,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__any","createdAt":1595896642202343868,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__auto_loan","createdAt":1595896642260798767,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__brokerage_account","createdAt":1595896639550130078,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__certificate_of_deposit","createdAt":1595896643210723905,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__checking","createdAt":1595896640763548235,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":42,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__credit_card","createdAt":1595896639734800167,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__home_equity_loan","createdAt":1595896642644368353,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__ira_investment","createdAt":1595896642076795921,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__line_of_credit","createdAt":1595896642071432710,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__loan","createdAt":1595896643101755055,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__money_market","createdAt":1595896642574933677,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":36,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__mortgage","createdAt":1595896640934767155,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__personal_loan","createdAt":1595896641378825138,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__roth_ira_investment","createdAt":1595896642821349899,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__savings","createdAt":1595896639534597334,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__simple_ira","createdAt":1595896639952200646,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__student_loan","createdAt":1595896642329394180,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__taxable_investment","createdAt":1595896641406468398,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"phone_recency","createdAt":1595896639708002303,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_count__brokerage_account","createdAt":1595896642501159753,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_cd_or_share_certificate","createdAt":1595896643269910233,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_checking_or_share_draft","createdAt":1595896640044097611,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_credit_card","createdAt":1595896641500609762,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_indirect_auto_loan","createdAt":1595896643353298900,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_line_of_credit","createdAt":1595896641219946523,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_loan","createdAt":1595896641263557904,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_money_market","createdAt":1595896642506172994,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_new_auto_loan","createdAt":1595896640791087766,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_real_estate_loan","createdAt":1595896642616627035,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_savings_or_share","createdAt":1595896642313967229,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_credit_card","createdAt":1595896642254165546,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_loan","createdAt":1595896642870853670,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_checking_or_share_draft","createdAt":1595896639333642847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_credit_card","createdAt":1595896642761030299,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_savings_or_share","createdAt":1595896643227769780,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_vehicle_loan","createdAt":1595896642991171391,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_used_auto_loan","createdAt":1595896642865599530,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_basic_checking_or_share_draft","createdAt":1595896640110059946,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_cd_or_share_certificate","createdAt":1595896640174468431,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_credit_card","createdAt":1595896640192571502,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_indirect_auto_loan","createdAt":1595896640563441892,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896643543919820,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_line_of_credit","createdAt":1595896640188449923,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_loan","createdAt":1595896639761090179,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_money_market","createdAt":1595896642731261748,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_new_auto_loan","createdAt":1595896643143947051,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_checking_or_share_draft","createdAt":1595896640346982139,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_savings_or_share","createdAt":1595896641346726397,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_real_estate_loan","createdAt":1595896643351578694,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_checking_or_share_draft","createdAt":1595896640302899735,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_savings_or_share","createdAt":1595896642726551847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_savings_or_share","createdAt":1595896642961872488,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_credit_card","createdAt":1595896640022927417,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_loan","createdAt":1595896642352438551,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_checking_or_share_draft","createdAt":1595896640804808345,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_credit_card","createdAt":1595896641113696262,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_savings_or_share","createdAt":1595896640312498993,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_vehicle_loan","createdAt":1595896639886308884,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_used_auto_loan","createdAt":1595896642412302198,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__debit_card","createdAt":1595896643127876947,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__education_savings_or_share","createdAt":1595896639338093281,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__escrow_account","createdAt":1595896642969874721,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__health_savings_account","createdAt":1595896640570458731,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_line_of_credit","createdAt":1595896642592520668,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_loan","createdAt":1595896643091627581,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__insurance_product","createdAt":1595896639744476133,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__interest_only_legal_trust_account","createdAt":1595896643005843869,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__membership_share","createdAt":1595896641732585653,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__other_service","createdAt":1595896641807551269,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_401k","createdAt":1595896639333271302,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":11,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_403b","createdAt":1595896643165638942,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_529","createdAt":1595896641640608805,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira","createdAt":1595896639555618615,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira_cd_or_share_certificate","createdAt":1595896641504460031,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__small_business_checking_or_share_draft","createdAt":1595896639332189957,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__special_indirect_vehicle_loan","createdAt":1595896641880199490,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__student_education_loan","createdAt":1595896639551558423,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira","createdAt":1595896643372890708,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira_cd_or_share_certificate","createdAt":1595896641621211665,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__trust","createdAt":1595896640709548779,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__vacation_club_holiday_savings","createdAt":1595896642256997553,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_recency__brokerage_account","createdAt":1595896641955724697,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_cd_or_share_certificate","createdAt":1595896642377298176,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_checking_or_share_draft","createdAt":1595896641640736297,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_credit_card","createdAt":1595896641199968081,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_indirect_auto_loan","createdAt":1595896643314414987,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_line_of_credit","createdAt":1595896641576267257,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_loan","createdAt":1595896641355763147,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_money_market","createdAt":1595896639970740640,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_new_auto_loan","createdAt":1595896642844117593,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_real_estate_loan","createdAt":1595896642953871689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_savings_or_share","createdAt":1595896643325231815,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_credit_card","createdAt":1595896642698454456,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_loan","createdAt":1595896641951407002,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_checking_or_share_draft","createdAt":1595896642711316173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_credit_card","createdAt":1595896642147545663,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_savings_or_share","createdAt":1595896642364919307,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_vehicle_loan","createdAt":1595896639876538030,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_used_auto_loan","createdAt":1595896643527381053,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_basic_checking_or_share_draft","createdAt":1595896639874545938,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_cd_or_share_certificate","createdAt":1595896643147292944,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_credit_card","createdAt":1595896642605033674,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_indirect_auto_loan","createdAt":1595896641097157897,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896642523867033,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_line_of_credit","createdAt":1595896641979726767,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_loan","createdAt":1595896639741458231,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_money_market","createdAt":1595896642350195677,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_new_auto_loan","createdAt":1595896642852653224,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_checking_or_share_draft","createdAt":1595896641391887646,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_savings_or_share","createdAt":1595896643501816261,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_real_estate_loan","createdAt":1595896643015250459,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_checking_or_share_draft","createdAt":1595896640413965217,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_savings_or_share","createdAt":1595896642440765022,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_savings_or_share","createdAt":1595896643523788186,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_credit_card","createdAt":1595896641228514181,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_loan","createdAt":1595896642815200189,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_checking_or_share_draft","createdAt":1595896639648900382,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_credit_card","createdAt":1595896639596071703,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_savings_or_share","createdAt":1595896642447703684,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_vehicle_loan","createdAt":1595896639863224318,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_used_auto_loan","createdAt":1595896639337315659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__debit_card","createdAt":1595896641091084110,"options":{"type":"int","base":0,"bitDepth":1,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__education_savings_or_share","createdAt":1595896643243122929,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__escrow_account","createdAt":1595896642288649376,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__health_savings_account","createdAt":1595896639713687734,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_line_of_credit","createdAt":1595896640644328270,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_loan","createdAt":1595896643005117660,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__insurance_product","createdAt":1595896640080338115,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__interest_only_legal_trust_account","createdAt":1595896640710544689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__membership_share","createdAt":1595896640920327282,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__other_service","createdAt":1595896641868683380,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_401k","createdAt":1595896643214447056,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_403b","createdAt":1595896639331510173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_529","createdAt":1595896641093053659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira","createdAt":1595896640406186395,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira_cd_or_share_certificate","createdAt":1595896642473602012,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__small_business_checking_or_share_draft","createdAt":1595896640631139451,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__special_indirect_vehicle_loan","createdAt":1595896641787887178,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__student_education_loan","createdAt":1595896640496522510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira","createdAt":1595896643383850510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira_cd_or_share_certificate","createdAt":1595896642145329304,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__trust","createdAt":1595896640506789797,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__vacation_club_holiday_savings","createdAt":1595896642206110046,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"stop_payment_recency","createdAt":1595896640954094741,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"survey_2053c210_adfc_4e14_afd3_9abc68a70719","createdAt":1595896642841258189,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2679bd7e_cdb8_4c24_a1da_e904799382eb","createdAt":1595896642498273724,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2f336120_7130_492b_b116_d93d560baa0a","createdAt":1595896639333711539,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_36e229ff_c0a6_48ad_928e_19b5abf69f11","createdAt":1595896642987004213,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa","createdAt":1595896643012024691,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796","createdAt":1595896640184709609,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_587ae933_b8e3_49a2_a23d_989df9ad119f","createdAt":1595896642862965911,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab","createdAt":1595896642292355510,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949","createdAt":1595896640788795677,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_9cebb459_162e_4518_88f6_bc1cdd630cd0","createdAt":1595896642343098690,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_a05de6d3_6620_4eb2_b9a2_511936680b57","createdAt":1595896643389135517,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_b570363c_ba94_4a7a_9891_141a1befc42b","createdAt":1595896643120229702,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_c358d6e0_4116_4230_84ed_869b315e89a1","createdAt":1595896642202109079,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_cce773ce_14e8_4fae_90cd_213a821a3bc0","createdAt":1595896640040535589,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_e49df858_4903_40ac_bd9f_96f764940f2c","createdAt":1595896642346035396,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9","createdAt":1595896640643626147,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"tablet_recency","createdAt":1595896640414042095,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"user_id","createdAt":1595896640846590883,"options":{"type":"int","base":0,"bitDepth":21,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"zip_code","createdAt":1595896640534832822,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}}],"shardWidth":1048576}]} diff --git a/cmd/sauron/sauron_test.go b/cmd/sauron/sauron_test.go deleted file mode 100644 index 53cf786de..000000000 --- a/cmd/sauron/sauron_test.go +++ /dev/null @@ -1,396 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "encoding/hex" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strconv" - "testing" - - pb "github.com/golang/protobuf/proto" //nolint:staticcheck - pbuf "github.com/molecula/go-pilosa/v2/gopilosa_pbuf" - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/encoding/proto" - "github.com/pkg/errors" -) - -func TestSauron(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //remove the existing db file if present - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - - schemaJson := `{"indexes":[{"name":"scratch","createdAt":1611185870149882000,"options":{"keys":false,"trackExistence":true}, "fields":[{"name":"luminosity","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}}]}]}` - - client := &http.Client{} - t.Run("Schema", func(t *testing.T) { - _, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) - panicOn(err) - // verify that we can fetch it back - resp, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp.Body) - panicOn(err) - sbody := string(body) - if sbody[:62] != schemaJson[:62] { - fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) - panic("unexpected schema back") - } - }) - // add a set field "color" - //POST localhost:10101/index/scratch/field/color - t.Run("Create Non-Keyed SetField", func(t *testing.T) { - resp, err := client.Post(url+"/index/scratch/field/rating", "application/text", bytes.NewBuffer(nil)) - panicOn(err) - if resp.StatusCode != 200 { - panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) - } - }) - t.Run("Create Keyed SetField", func(t *testing.T) { - resp, err := client.Post(url+"/index/scratch/field/color", "application/text", bytes.NewBuffer([]byte(`{"options": {"keys": true}}`))) - panicOn(err) - if resp.StatusCode != 200 { - panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) - } - }) - t.Run("Create Time Field", func(t *testing.T) { - resp, err := client.Post(url+"/index/scratch/field/event", "application/text", bytes.NewBuffer([]byte(`{"options": { "type": "time", "timeQuantum": "YMDH" }}`))) - panicOn(err) - if resp.StatusCode != 200 { - panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) - } - }) - - // set some bits - - // set some bits - - t.Run("Set", func(t *testing.T) { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) - panicOn(err) - // test for dups - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) - panicOn(err) - // - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=red)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=red)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=green)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=green)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=green)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=green)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=yellow)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=yellow)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=yellow)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=yellow)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=orange)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=orange)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=orange)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=orange)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, luminosity=1)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, luminosity=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, luminosity=4)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, luminosity=5)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, luminosity=4)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, luminosity=3)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, luminosity=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, luminosity=1)`)) - panicOn(err) - - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, rating=1)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(10, rating=1)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, rating=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, rating=2)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, rating=3)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, rating=4)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, rating=4)`)) - panicOn(err) - - //time fields - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T01:00)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T02:00)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, event=1,2021-02-05T02:05)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, event=1,2021-02-05T03:00)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, event=1,2021-02-05T04:00)`)) - panicOn(err) - }) - t.Run("Clear", func(t *testing.T) { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Clear(1, color=purple)`)) - panicOn(err) - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`ClearRow(color=purple)`)) - panicOn(err) - }) - t.Run("Store", func(t *testing.T) { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Store(Row(color=red), color=purple)`)) - panicOn(err) - }) - // verify that we can fetch it back - var schema pilosa.Schema - t.Run("Fetch Schema", func(t *testing.T) { - resp, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp.Body) - panicOn(err) - err = json.Unmarshal(body, &schema) - if err != nil { - panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) - } - }) - t.Run("Import Roaring", func(t *testing.T) { - //sets 12 bits in shard 0 - roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - idx := schema.Indexes[0] - var fld *pilosa.FieldInfo - for _, f := range idx.Fields { - if f.Name == "rating" { - fld = f - } - } - if fld == nil { - panic("field color not found") - } - msg := pilosa.ImportRoaringRequest{ - IndexCreatedAt: idx.CreatedAt, - FieldCreatedAt: fld.CreatedAt, - Clear: false, - Views: map[string][]byte{ - "": roaringData, - }, - UpdateExistence: true, - } - ser := proto.Serializer{} - data, err := ser.Marshal(&msg) - if err != nil { - t.Fatal(err) - } - furl := url + "/index/scratch/field/rating/import-roaring/0" - req, err := http.NewRequest("POST", furl, bytes.NewBuffer(data)) - panicOn(err) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - - // maybe need to Accept: "application/x-protobuf" - resp, err := client.Do(req) - panicOn(err) - _ = resp - }) - t.Run("Read Query", func(t *testing.T) { - for _, tst := range GetTests() { - _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(tst.Pql)) - panicOn(err) - } - }) -} - -func makeImportRequest(field *FieldInfo2, shard uint64, rows, columns []uint64, clear bool) (path string, data []byte, err error) { - msg := &pbuf.ImportRequest{ - Index: field.GetIndexName(), - IndexCreatedAt: field.GetIndexCreatedAt(), - Field: field.GetName(), - FieldCreatedAt: field.GetCreatedAt(), - Shard: shard, - RowIDs: rows, - ColumnIDs: columns, - } - data, err = pb.Marshal(msg) - if err != nil { - return "", nil, errors.Wrap(err, "marshaling Import to protobuf") - } - path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) - return path, data, nil -} -func makeImportKeyRequest(field *FieldInfo2, shard uint64, rows []string, columns []uint64, clear bool) (path string, data []byte, err error) { - msg := &pbuf.ImportRequest{ - Index: field.GetIndexName(), - IndexCreatedAt: field.GetIndexCreatedAt(), - Field: field.GetName(), - FieldCreatedAt: field.GetCreatedAt(), - Shard: shard, - RowKeys: rows, - ColumnIDs: columns, - } - data, err = pb.Marshal(msg) - if err != nil { - return "", nil, errors.Wrap(err, "marshaling Import to protobuf") - } - path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) - return path, data, nil -} - -func makeImportValueRequest(field *FieldInfo2, shard uint64, values []int64, columns []uint64, clear bool) (path string, data []byte, err error) { - msg := &pbuf.ImportValueRequest{ - Index: field.GetIndexName(), - IndexCreatedAt: field.GetIndexCreatedAt(), - Field: field.GetName(), - FieldCreatedAt: field.GetCreatedAt(), - Shard: shard, - Values: values, - ColumnIDs: columns, - } - data, err = pb.Marshal(msg) - if err != nil { - return "", nil, errors.Wrap(err, "marshaling Import to protobuf") - } - path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) - return path, data, nil -} -func TestSauronImport(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //remove the existing db file if present - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - schemaJson := `{ - "indexes": [{ - "name": "scratch", - "createdAt": 1611185870149882000, - "options": { - "keys": false, - "trackExistence": true - }, - "fields": [{ - "name": "bsi", - "createdAt": 1595896639332730413, - "options": { - "type": "int", - "base": 0, - "bitDepth": 31, - "min": -9223372036854775808, - "max": 9223372036854775807, - "keys": false, - "foreignIndex": "" - } - }, { - "name": "decimal", - "createdAt": 1613177295654228860, - "options": { - "type": "decimal", - "base": 0, - "scale": 1, - "bitDepth": 0, - "min": -922337203685477580.8, - "max": 922337203685477580.7, - "keys": false - } - },{ - "name": "setkeyfield", - "createdAt": 1613345793598357200, - "options": { - "type": "set", - "cacheType": "ranked", - "cacheSize": 50000, - "keys": true - } - },{ - "name": "setfield", - "createdAt": 1613345793598357200, - "options": { - "type": "set", - "cacheType": "ranked", - "cacheSize": 50000, - "keys": false - } - }] - }] -}` - client := &http.Client{} - _, err = client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) - panicOn(err) - - t.Run("basic row/column", func(t *testing.T) { - f, err := proxy.sql.GetField("scratch", "setfield") - panicOn(err) - clear := false - cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} - rows := []uint64{0, 0, 0, 1, 1, 1, 2, 2, 2} - shard := uint64(0) - path, payload, err := makeImportRequest(f, shard, rows, cols, clear) - _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) - panicOn(err) - }) - t.Run("basic rowkey/column", func(t *testing.T) { - f, err := proxy.sql.GetField("scratch", "setkeyfield") - panicOn(err) - clear := false - cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} - rows := []string{"red", "red", "red", "blue", "blue", "blue", "green", "green", "green"} - shard := uint64(0) - path, payload, err := makeImportKeyRequest(f, shard, rows, cols, clear) - _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) - panicOn(err) - }) - t.Run("basic import values", func(t *testing.T) { - f, err := proxy.sql.GetField("scratch", "bsi") - panicOn(err) - clear := false - cols := []uint64{0, 1, 2, 3, 4} - values := []int64{40, 30, 20, 10, 5} - shard := uint64(0) - path, payload, err := makeImportValueRequest(f, shard, values, cols, clear) - _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) - panicOn(err) - }) - -} diff --git a/cmd/sauron/schema_test.go b/cmd/sauron/schema_test.go deleted file mode 100644 index 23f5e924b..000000000 --- a/cmd/sauron/schema_test.go +++ /dev/null @@ -1,353 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strings" - "testing" - - "github.com/pilosa/pilosa/v2" -) - -func NewSauronTestConfig(t *testing.T) *SauronConfig { - - bind := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) - bindGRPC := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) - gossipPort := fmt.Sprintf("%d", GetAvailPort()) - dataDir, err := ioutil.TempDir("", "pilosa-sauron-target-*") - panicOn(err) - - return &SauronConfig{ - DatabaseName: strings.ToLower(t.Name()), - Bind: bind, - BindGRPC: bindGRPC, - GossipPort: gossipPort, - DataDir: dataDir, - } -} - -func TestProxyCanHandlePostSchema(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //start with - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - - // prep for test by cleaning up leftovers from any old run. - //panicOn(proxy.sql.DropTablesWithPrefix(t.Name())) - - //schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } }, { "name ": "color", "options ": {} } ] }] }` - /* - schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } } ] }] }` - - - vv("Posting err = '%v'", schemaJson) - resp, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) - if resp.StatusCode != http.StatusNoContent { - panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) - } - // verify that we can fetch it back - resp2, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp2.Body) - panicOn(err) - sbody := string(body) - vv("body = '%v'", sbody) - if sbody[:62] != schemaJson[:62] { - fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) - panic("unxpected schema back") - } - */ - // load large schema too - client := &http.Client{} - schm, err := ioutil.ReadFile("sample_schema.json") - panicOn(err) - - resp, err := client.Post(url+"/schema", "application/json", bytes.NewBuffer(schm)) - panicOn(err) - if resp.StatusCode != http.StatusNoContent { - panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) - } - - // verify all tables expected are present - obs := make(map[string]bool) - for i, table := range proxy.sql.ListIndexFields() { - _ = i - obs[table] = true - if !expectedTables[table] { - panic(fmt.Sprintf("observed table '%v' but was not expected", table)) - } - } - for table := range expectedTables { - if !obs[table] { - panic(fmt.Sprintf("expected table '%v' but was not observed", table)) - } - } - -} - -func TestProxyCanCreateIndexFieldViaPost(t *testing.T) { - - cfg := NewSauronTestConfig(t) - proxy := NewSauron(cfg) - proxy.sql.MustRemove() //start with - url, err := proxy.Start() - panicOn(err) - defer proxy.Stop() - client := &http.Client{} - //create non-keyed or "simple" index - resp, err := client.Post(url+"/index/simple", "application/json", bytes.NewBuffer(nil)) - panicOn(err) - if resp.StatusCode != http.StatusOK { - panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) - } - - fields := []struct { - name string - options []byte - }{ - {name: "set", options: []byte{}}, - {name: "keyset", options: []byte{}}, - {name: "bsi", options: []byte{}}, - } - for _, field := range fields { - //create set field - resp, err = client.Post(fmt.Sprintf("%v/index/simple/field/%v", url, field.name), "application/json", bytes.NewBuffer(field.options)) - panicOn(err) - if resp.StatusCode != http.StatusOK { - panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) - } - } - // verify that we can fetch it back - resp2, err := client.Get(url + "/schema") - panicOn(err) - body, err := ioutil.ReadAll(resp2.Body) - panicOn(err) - var schema pilosa.Schema - err = json.Unmarshal(body, &schema) - if err != nil { - panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) - } - - expected := map[string]map[string]bool{ - "simple": {"bsi": true, "set": true, "keyset": true}, - } - for _, i := range schema.Indexes { - ll, ok := expected[i.Name] - if !ok { - panic(fmt.Sprintf("expected index not present: %v", i.Name)) - } - for _, f := range i.Fields { - _, ok := ll[f.Name] - if !ok { - panic(fmt.Sprintf("expected field '%v' not present in index'%v'", f.Name, i.Name)) - } - } - } -} - -var expectedTables = map[string]bool{ - "trait_store/aba": true, - "trait_store/ach_batch_recency": true, - "trait_store/ach_pass_thru_recency": true, - "trait_store/ach_payment_recency": true, - "trait_store/bools": true, - "trait_store/bools-exists": true, - "trait_store/central_group": true, - "trait_store/custom_audiences": true, - "trait_store/days_since_last_logon": true, - "trait_store/db": true, - "trait_store/desktop_recency": true, - "trait_store/domestic_wire_recency": true, - "trait_store/external_transfer_recency": true, - "trait_store/fields": true, - "trait_store/funds_transfer_recency": true, - "trait_store/international_wire_recency": true, - "trait_store/mobile_remote_deposit_recency": true, - "trait_store/payroll_recency": true, - "trait_store/pfm_category_total_current_balance__401k_investment": true, - "trait_store/pfm_category_total_current_balance__403b_investment": true, - "trait_store/pfm_category_total_current_balance__529_investment": true, - "trait_store/pfm_category_total_current_balance__any": true, - "trait_store/pfm_category_total_current_balance__auto_loan": true, - "trait_store/pfm_category_total_current_balance__brokerage_account": true, - "trait_store/pfm_category_total_current_balance__certificate_of_deposit": true, - "trait_store/pfm_category_total_current_balance__checking": true, - "trait_store/pfm_category_total_current_balance__credit_card": true, - "trait_store/pfm_category_total_current_balance__home_equity_loan": true, - "trait_store/pfm_category_total_current_balance__ira_investment": true, - "trait_store/pfm_category_total_current_balance__line_of_credit": true, - "trait_store/pfm_category_total_current_balance__loan": true, - "trait_store/pfm_category_total_current_balance__money_market": true, - "trait_store/pfm_category_total_current_balance__mortgage": true, - "trait_store/pfm_category_total_current_balance__personal_loan": true, - "trait_store/pfm_category_total_current_balance__roth_ira_investment": true, - "trait_store/pfm_category_total_current_balance__savings": true, - "trait_store/pfm_category_total_current_balance__simple_ira": true, - "trait_store/pfm_category_total_current_balance__student_loan": true, - "trait_store/pfm_category_total_current_balance__taxable_investment": true, - "trait_store/phone_recency": true, - "trait_store/product_count__brokerage_account": true, - "trait_store/product_count__commercial_cd_or_share_certificate": true, - "trait_store/product_count__commercial_checking_or_share_draft": true, - "trait_store/product_count__commercial_credit_card": true, - "trait_store/product_count__commercial_indirect_auto_loan": true, - "trait_store/product_count__commercial_line_of_credit": true, - "trait_store/product_count__commercial_loan": true, - "trait_store/product_count__commercial_money_market": true, - "trait_store/product_count__commercial_new_auto_loan": true, - "trait_store/product_count__commercial_real_estate_loan": true, - "trait_store/product_count__commercial_savings_or_share": true, - "trait_store/product_count__commercial_secured_credit_card": true, - "trait_store/product_count__commercial_secured_loan": true, - "trait_store/product_count__commercial_special_checking_or_share_draft": true, - "trait_store/product_count__commercial_special_credit_card": true, - "trait_store/product_count__commercial_special_savings_or_share": true, - "trait_store/product_count__commercial_special_vehicle_loan": true, - "trait_store/product_count__commercial_used_auto_loan": true, - "trait_store/product_count__consumer_basic_checking_or_share_draft": true, - "trait_store/product_count__consumer_cd_or_share_certificate": true, - "trait_store/product_count__consumer_credit_card": true, - "trait_store/product_count__consumer_indirect_auto_loan": true, - "trait_store/product_count__consumer_interest_bearing_checking_or_share_draft": true, - "trait_store/product_count__consumer_line_of_credit": true, - "trait_store/product_count__consumer_loan": true, - "trait_store/product_count__consumer_money_market": true, - "trait_store/product_count__consumer_new_auto_loan": true, - "trait_store/product_count__consumer_premium_checking_or_share_draft": true, - "trait_store/product_count__consumer_premium_savings_or_share": true, - "trait_store/product_count__consumer_real_estate_loan": true, - "trait_store/product_count__consumer_restricted_checking_or_share_draft": true, - "trait_store/product_count__consumer_restricted_savings_or_share": true, - "trait_store/product_count__consumer_savings_or_share": true, - "trait_store/product_count__consumer_secured_credit_card": true, - "trait_store/product_count__consumer_secured_loan": true, - "trait_store/product_count__consumer_special_checking_or_share_draft": true, - "trait_store/product_count__consumer_special_credit_card": true, - "trait_store/product_count__consumer_special_savings_or_share": true, - "trait_store/product_count__consumer_special_vehicle_loan": true, - "trait_store/product_count__consumer_used_auto_loan": true, - "trait_store/product_count__debit_card": true, - "trait_store/product_count__education_savings_or_share": true, - "trait_store/product_count__escrow_account": true, - "trait_store/product_count__health_savings_account": true, - "trait_store/product_count__home_equity_line_of_credit": true, - "trait_store/product_count__home_equity_loan": true, - "trait_store/product_count__insurance_product": true, - "trait_store/product_count__interest_only_legal_trust_account": true, - "trait_store/product_count__membership_share": true, - "trait_store/product_count__other_service": true, - "trait_store/product_count__plan_401k": true, - "trait_store/product_count__plan_403b": true, - "trait_store/product_count__plan_529": true, - "trait_store/product_count__roth_ira": true, - "trait_store/product_count__roth_ira_cd_or_share_certificate": true, - "trait_store/product_count__small_business_checking_or_share_draft": true, - "trait_store/product_count__special_indirect_vehicle_loan": true, - "trait_store/product_count__student_education_loan": true, - "trait_store/product_count__traditional_ira": true, - "trait_store/product_count__traditional_ira_cd_or_share_certificate": true, - "trait_store/product_count__trust": true, - "trait_store/product_count__vacation_club_holiday_savings": true, - "trait_store/product_recency__brokerage_account": true, - "trait_store/product_recency__commercial_cd_or_share_certificate": true, - "trait_store/product_recency__commercial_checking_or_share_draft": true, - "trait_store/product_recency__commercial_credit_card": true, - "trait_store/product_recency__commercial_indirect_auto_loan": true, - "trait_store/product_recency__commercial_line_of_credit": true, - "trait_store/product_recency__commercial_loan": true, - "trait_store/product_recency__commercial_money_market": true, - "trait_store/product_recency__commercial_new_auto_loan": true, - "trait_store/product_recency__commercial_real_estate_loan": true, - "trait_store/product_recency__commercial_savings_or_share": true, - "trait_store/product_recency__commercial_secured_credit_card": true, - "trait_store/product_recency__commercial_secured_loan": true, - "trait_store/product_recency__commercial_special_checking_or_share_draft": true, - "trait_store/product_recency__commercial_special_credit_card": true, - "trait_store/product_recency__commercial_special_savings_or_share": true, - "trait_store/product_recency__commercial_special_vehicle_loan": true, - "trait_store/product_recency__commercial_used_auto_loan": true, - "trait_store/product_recency__consumer_basic_checking_or_share_draft": true, - "trait_store/product_recency__consumer_cd_or_share_certificate": true, - "trait_store/product_recency__consumer_credit_card": true, - "trait_store/product_recency__consumer_indirect_auto_loan": true, - "trait_store/product_recency__consumer_interest_bearing_checking_or_share_draft": true, - "trait_store/product_recency__consumer_line_of_credit": true, - "trait_store/product_recency__consumer_loan": true, - "trait_store/product_recency__consumer_money_market": true, - "trait_store/product_recency__consumer_new_auto_loan": true, - "trait_store/product_recency__consumer_premium_checking_or_share_draft": true, - "trait_store/product_recency__consumer_premium_savings_or_share": true, - "trait_store/product_recency__consumer_real_estate_loan": true, - "trait_store/product_recency__consumer_restricted_checking_or_share_draft": true, - "trait_store/product_recency__consumer_restricted_savings_or_share": true, - "trait_store/product_recency__consumer_savings_or_share": true, - "trait_store/product_recency__consumer_secured_credit_card": true, - "trait_store/product_recency__consumer_secured_loan": true, - "trait_store/product_recency__consumer_special_checking_or_share_draft": true, - "trait_store/product_recency__consumer_special_credit_card": true, - "trait_store/product_recency__consumer_special_savings_or_share": true, - "trait_store/product_recency__consumer_special_vehicle_loan": true, - "trait_store/product_recency__consumer_used_auto_loan": true, - "trait_store/product_recency__debit_card": true, - "trait_store/product_recency__education_savings_or_share": true, - "trait_store/product_recency__escrow_account": true, - "trait_store/product_recency__health_savings_account": true, - "trait_store/product_recency__home_equity_line_of_credit": true, - "trait_store/product_recency__home_equity_loan": true, - "trait_store/product_recency__insurance_product": true, - "trait_store/product_recency__interest_only_legal_trust_account": true, - "trait_store/product_recency__membership_share": true, - "trait_store/product_recency__other_service": true, - "trait_store/product_recency__plan_401k": true, - "trait_store/product_recency__plan_403b": true, - "trait_store/product_recency__plan_529": true, - "trait_store/product_recency__roth_ira": true, - "trait_store/product_recency__roth_ira_cd_or_share_certificate": true, - "trait_store/product_recency__small_business_checking_or_share_draft": true, - "trait_store/product_recency__special_indirect_vehicle_loan": true, - "trait_store/product_recency__student_education_loan": true, - "trait_store/product_recency__traditional_ira": true, - "trait_store/product_recency__traditional_ira_cd_or_share_certificate": true, - "trait_store/product_recency__trust": true, - "trait_store/product_recency__vacation_club_holiday_savings": true, - "trait_store/stop_payment_recency": true, - "trait_store/survey_2053c210_adfc_4e14_afd3_9abc68a70719": true, - "trait_store/survey_2679bd7e_cdb8_4c24_a1da_e904799382eb": true, - "trait_store/survey_2f336120_7130_492b_b116_d93d560baa0a": true, - "trait_store/survey_36e229ff_c0a6_48ad_928e_19b5abf69f11": true, - "trait_store/survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa": true, - "trait_store/survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796": true, - "trait_store/survey_587ae933_b8e3_49a2_a23d_989df9ad119f": true, - "trait_store/survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab": true, - "trait_store/survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949": true, - "trait_store/survey_9cebb459_162e_4518_88f6_bc1cdd630cd0": true, - "trait_store/survey_a05de6d3_6620_4eb2_b9a2_511936680b57": true, - "trait_store/survey_b570363c_ba94_4a7a_9891_141a1befc42b": true, - "trait_store/survey_c358d6e0_4116_4230_84ed_869b315e89a1": true, - "trait_store/survey_cce773ce_14e8_4fae_90cd_213a821a3bc0": true, - "trait_store/survey_e49df858_4903_40ac_bd9f_96f764940f2c": true, - "trait_store/survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9": true, - "trait_store/tablet_recency": true, - "trait_store/user_id": true, - "trait_store/zip_code": true, -} diff --git a/cmd/sauron/sqlgen_test.go b/cmd/sauron/sqlgen_test.go deleted file mode 100644 index b9d11a728..000000000 --- a/cmd/sauron/sqlgen_test.go +++ /dev/null @@ -1,260 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/v2/pql" - "github.com/shurcooL/go-goon" -) - -/* -sqlite db this test validated this against -create table bits (field ,row , column , timestamp); - create table λbsi (field string,column bigint , val bigint); - CREATE VIEW columns as - select distinct column from λbits - UNION - select distinct column from λbsi; - - INSERT INTO λbits VALUES( 'color','red',1); - INSERT INTO λbits VALUES( 'color','red',2); - INSERT INTO λbits VALUES( 'color','red',3); - INSERT INTO λbits VALUES( 'color','red',4); - INSERT INTO λbits VALUES( 'color','red',5); - INSERT INTO λbits VALUES( 'color','green',2); - INSERT INTO λbits VALUES( 'color','green',4); - INSERT INTO λbits VALUES( 'color','green',6); - INSERT INTO λbits VALUES( 'color','green',8); - INSERT INTO λbits VALUES( 'color','yellow',1); - INSERT INTO λbits VALUES( 'color','yellow',3); - INSERT INTO λbits VALUES( 'color','yellow',5); - INSERT INTO λbits VALUES( 'color','yellow',7); - INSERT INTO λbits VALUES( 'color','orange',5); - INSERT INTO λbits VALUES( 'color','orange',6); - INSERT INTO λbits VALUES( 'color','orange',7); - INSERT INTO λbits VALUES( 'color','orange',8); - - INSERT INTO λbits VALUES( 'rating',1,8); - INSERT INTO λbits VALUES( 'rating',1,7); - INSERT INTO λbits VALUES( 'rating',1,10); - INSERT INTO λbits VALUES( 'rating',2,6); - INSERT INTO λbits VALUES( 'rating',2,5); - INSERT INTO λbits VALUES( 'rating',3,4); - INSERT INTO λbits VALUES( 'rating',4,3); - INSERT INTO λbits VALUES( 'rating',4,2); - - INSERT INTO λbsi VALUES( 'luminosity',1, 1); - INSERT INTO λbsi VALUES( 'luminosity',2, 2); - INSERT INTO λbsi VALUES( 'luminosity',3, 4); - INSERT INTO λbsi VALUES( 'luminosity',4, 5); - INSERT INTO λbsi VALUES( 'luminosity',5, 4); - INSERT INTO λbsi VALUES( 'luminosity',6, 3); - INSERT INTO λbsi VALUES( 'luminosity',7, 2); - INSERT INTO λbsi VALUES( 'luminosity',8, 1); -*/ -func GetTests() []struct { - Pql string - Sql string -} { - return []struct { - Pql string - Sql string - }{ - { - Pql: "Row(rating=0)", - Sql: `select distinct column from λbits where field="rating" AND row="0"`, - }, - { - Pql: "Row(color=red)", - Sql: `select distinct column from λbits where field="color" AND row="red"`, - }, - { - Pql: "Row(luminosity>2)", - Sql: `select distinct column from λbsi where field="luminosity" and val>2`, - }, - { - Pql: "Row(luminosity<2)", - Sql: `select distinct column from λbsi where field="luminosity" and val<2`, - }, - { - Pql: "Row(luminosity==4)", - Sql: `select distinct column from λbsi where field="luminosity" and val=4`, - }, - { - Pql: "Row(luminosity<=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val<=4`, - }, - { - Pql: "Row(luminosity>=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val>=4`, - }, - { - Pql: "Row(luminosity!=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val!=4`, - }, - { - Pql: "Row(2<=luminosity<=4)", - Sql: `select distinct column from λbsi where field="luminosity" and val>=2 and val<=4`, - }, - { - Pql: "Row(22 and val<=4`, - }, - { - Pql: "Row(22 and val<4`, - }, - { - Pql: `Row(event=1, from='2021-02-01T00:00', to='2021-02-05T03:00')`, - Sql: `select distinct column from λbits where field="event" AND row="1" AND timestamp >= "2021-02-01 00:00" AND timestamp < "2021-02-05 03:00"`, - }, - { - Pql: "Count(row(color=red))", - Sql: `select count(*) from( select distinct column from λbits where field="color" AND row="red" )`, - }, - { - Pql: `Intersect(Row(color=red),Row(color=green))`, - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: "Count(Union(Row(color=red),Row(color=green)))", - Sql: `select count(*) from( select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") )`, - }, - { - Pql: "Union(Row(color=red),Row(color=green))", - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: "Difference(Row(color=red),Row(color=green))", - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" except select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: `All()`, - Sql: `select column from λcolumns`, - }, - { - Pql: "Not(Row(color=red))", - Sql: `select column from λcolumns except select distinct column from λbits where field="color" AND row="red"`, - }, - { - Pql: "Xor(Row(color=red),Row(color=green))", - Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") except select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, - }, - { - Pql: "Distinct(field=luminosity)", - Sql: `select distinct val from λbsi where field="luminosity"`, - }, - { - Pql: "Distinct(Row(color=red),field=luminosity)", - Sql: `select distinct val from λbsi where field="luminosity" AND column in (select distinct column from λbits where field="color" AND row="red")`, - }, - { - Pql: "Distinct(Intersect(Row(color=red),Row(color=green)),field=luminosity)", - Sql: `select distinct val from λbsi where field="luminosity" AND column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green"))`, - }, - { - Pql: "Rows(color)", - Sql: `select distinct field, row from λbits where field="color" order by row`, - }, - { - Pql: "Rows(color, limit=10)", - Sql: `select distinct field, row from λbits where field="color" order by row limit 10`, - }, - { - Pql: "Rows(color, column=1)", - Sql: `select distinct field, row from λbits where field="color" and column='1' order by row`, - }, - // { // TODO (twg) Not supported by this test, can't reliably get key ordering - // Pql: "Rows(color, previous=green )", - // Sql: `select distinct field, row from λbits where field="color" and row>'green' order by row`, - // }, - { - Pql: "GroupBy(Rows(color))", - Sql: `select f1.field, f1.row,count(*) as cnt from λbits f1 where f1.field='color' group by f1.field,f1.row order by cnt desc`, - }, - { - Pql: "GroupBy(Rows(color),Rows(rating))", - Sql: `select f1.field, f1.row,f2.field, f2.row,count(*) as cnt from λbits f1 inner join λbits f2 on f1.column = f2.column where f1.field='color'and f2.field='rating' group by f1.field,f1.row,f2.field,f2.row order by cnt desc`, - }, - { - Pql: "TopN(color)", - Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, - }, - { - Pql: "TopN(color,n=2)", - Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc limit 2`, - }, - { - Pql: "TopN(color,Row(color=red),n=2)", - Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="color" AND row="red") group by row order by cnt desc limit 2`, - }, - { - Pql: "Topk(color)", - Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, - }, - { - Pql: "TopK(color, k=2, filter=Row(rating=1))", - Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="rating" AND row="1") group by row order by cnt desc limit 2`, - }, - { - Pql: `Min(field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity")`, - }, - { - Pql: `Max(field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select max(val) from λbsi where field="luminosity")`, - }, - { - Pql: `Min(Row(color=orange),field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select distinct column from λbits where field="color" AND row="orange")) and column in (select distinct column from λbits where field="color" AND row="orange")`, - }, - { - Pql: `Min(Intersect(Row(color=red),Row(color=orange)),field="luminosity")`, - Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))) and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))`, - }, - { - Pql: `Sum(field="luminosity")`, - Sql: `select sum(val),count(*) from λbsi where field="luminosity"`, - }, - { - Pql: `ConstRow(columns=[1,3,5])`, - Sql: `select column1 as column from (values (1),(3),(5))`, - }, - { - Pql: `UnionRows(Rows(color))`, - Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, - }, - { - Pql: `UnionRows(Rows(color),Rows(rating))`, - Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, - }, - { - Pql: `IncludesColumn(Row(color=red), column=1)`, - Sql: `select count(*)>0 from (select column from(select distinct column from λbits where field="color" AND row="red") where column='1')`, - }, - { - Pql: `Extract(Row(color=red),Rows(color),Rows(rating))`, - Sql: `select sq.field, sq.row, b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row where b.column in (select distinct column from λbits where field="color" AND row="red")`, - }, - { - Pql: `Extract(ConstRow(columns=[1,2,3]))`, - Sql: `select '' as field, '' as row,column from (select column1 as column from (values (1),(2),(3)))`, - }, - } -} -func TestParse(t *testing.T) { - - for _, tst := range GetTests() { - - q, err := pql.NewParser(strings.NewReader(tst.Pql)).Parse() - panicOn(err) - sql, _, _, err := ToSql(q.Calls, "") - panicOn(err) - if sql != tst.Sql { - vv("\npql:%v\ngenerated:\n%v\nexpected:\n%v\n", tst.Pql, sql, tst.Sql) - goon.Dump(q) - } - - } -} diff --git a/cmd/sauron/test_sqlite.sh b/cmd/sauron/test_sqlite.sh deleted file mode 100755 index d0ab85b2f..000000000 --- a/cmd/sauron/test_sqlite.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -db="poc2.db" -sqlite3 ${db} < 0 { - frames := runtime.CallersFrames(pc[:n]) - for i := 0; i <= target; i++ { - contender, more := frames.Next() - if i == target { - f = contender - } - if !more { - break - } - } - } - return f.Function -} diff --git a/go.mod b/go.mod index dcd6b603a..a9a8ef35d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d - github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 + github.com/DataDog/datadog-go v2.2.0+incompatible + github.com/OneOfOne/xxhash v1.2.5 // indirect + github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/beevik/ntp v0.3.0 github.com/benbjohnson/immutable v0.3.0 github.com/cespare/xxhash v1.1.0 @@ -15,26 +17,30 @@ require ( github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 - github.com/gogo/protobuf v1.2.1 + github.com/gogo/protobuf v1.3.1 github.com/golang/protobuf v1.4.2 github.com/google/go-cmp v0.5.2 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect - github.com/gorilla/handlers v1.3.0 - github.com/gorilla/mux v1.7.0 - github.com/hashicorp/memberlist v0.1.3 + github.com/gorilla/handlers v1.4.1 + github.com/gorilla/mux v1.7.3 + github.com/hashicorp/go-immutable-radix v1.1.0 // indirect + github.com/hashicorp/go-msgpack v0.5.5 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/memberlist v0.1.4 github.com/improbable-eng/grpc-web v0.13.0 github.com/kr/text v0.2.0 // indirect github.com/lib/pq v1.8.0 + github.com/miekg/dns v1.1.15 // indirect github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/pelletier/go-toml v1.2.0 + github.com/pelletier/go-toml v1.4.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.1.0 + github.com/prometheus/procfs v0.0.3 // indirect github.com/prometheus/prom2json v1.3.0 github.com/rakyll/statik v0.1.7 - github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/rs/cors v1.7.0 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil/v3 v3.20.11 @@ -54,10 +60,11 @@ require ( golang.org/x/sys v0.0.0-20201214095126-aec9a390925b // indirect golang.org/x/text v0.3.3 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/grpc v1.28.0 + google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 // indirect + google.golang.org/grpc v1.29.1 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.3.0 // indirect - modernc.org/mathutil v1.0.0 + modernc.org/mathutil v1.2.2 modernc.org/strutil v1.0.0 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible ) diff --git a/go.sum b/go.sum index f1ad4ecc5..2b9010aac 100644 --- a/go.sum +++ b/go.sum @@ -16,10 +16,11 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= -github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= -github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/DataDog/datadog-go v2.2.0+incompatible h1:V5BKkxACZLjzHjSgBbr2gvLA2Ae49yhc6CSY7MLy5k4= +github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= +github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -28,8 +29,9 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= @@ -43,6 +45,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -64,10 +68,10 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -86,8 +90,9 @@ github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -97,7 +102,6 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -112,7 +116,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -125,10 +128,10 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= -github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= -github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/handlers v1.4.1 h1:BHvcRGJe/TrL+OqFxoKQGddTgeibiOjaBssV5a/N9sw= +github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -138,23 +141,26 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= +github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc= +github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -174,15 +180,13 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -194,13 +198,14 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= +github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -217,10 +222,12 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -231,6 +238,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -240,27 +248,31 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y= github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= -github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= -github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= @@ -293,17 +305,16 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= @@ -403,7 +414,6 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214095126-aec9a390925b h1:tv7/y4pd+sR8bcNb2D6o7BNU6zjWm0VjQLac+w7fNNM= @@ -417,6 +427,7 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -455,18 +466,19 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU= +google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -485,7 +497,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -498,8 +509,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/http/client_test.go b/http/client_test.go index 87a010501..bb11609d4 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -1462,6 +1462,9 @@ func TestClient_ImportRoaringExists(t *testing.T) { t.Fatal(err) } qr, err := node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + if err != nil { + t.Fatalf(" %v ", err) + } got := qr.Results[0].(*pilosa.Row).Columns() if !reflect.DeepEqual(got, []uint64{}) { t.Fatalf(" Row unexpected columns: got %+v expected: %+v", got, []uint64{}) @@ -1473,6 +1476,9 @@ func TestClient_ImportRoaringExists(t *testing.T) { expected := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537} qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + if err != nil { + t.Fatalf("Query error: %+v", err) + } got = qr.Results[0].(*pilosa.Row).Columns() if !reflect.DeepEqual(got, expected) { t.Fatalf("All unexpected columns: got %+v expected: %+v", got, expected) diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 4773ef666..5794a05d0 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -16,9 +16,10 @@ package pilosa import ( "bytes" - "github.com/pilosa/pilosa/v2/roaring" "reflect" "testing" + + "github.com/pilosa/pilosa/v2/roaring" ) func TestValidateName(t *testing.T) { @@ -73,22 +74,32 @@ func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, e func TestAPI_CombineForExistence(t *testing.T) { bm := roaring.NewBitmap() - bm.Add(pos(1, 1)) - bm.Add(pos(1, 2)) - bm.Add(pos(1, 65537)) //make sure to cross container boundary - bm.Add(pos(1, 65538)) - bm.Add(pos(2, 1)) - bm.Add(pos(2, 2)) - bm.Add(pos(2, 65537)) - bm.Add(pos(2, 65538)) + _, err := bm.Add(pos(1, 1)) + panicOn(err) + _, err = bm.Add(pos(1, 2)) + panicOn(err) + _, err = bm.Add(pos(1, 65537)) //make sure to cross container boundary + panicOn(err) + _, err = bm.Add(pos(1, 65538)) + panicOn(err) + _, err = bm.Add(pos(2, 1)) + panicOn(err) + _, err = bm.Add(pos(2, 2)) + panicOn(err) + _, err = bm.Add(pos(2, 65537)) + panicOn(err) + _, err = bm.Add(pos(2, 65538)) + panicOn(err) buf := new(bytes.Buffer) - _, err := bm.WriteTo(buf) + _, err = bm.WriteTo(buf) panicOn(err) raw := buf.Bytes() - results := combineForExistence(raw) + results, err := combineForExistence(raw) + panicOn(err) bm2 := roaring.NewBitmap() - bm2.ImportRoaringBits(results, false, false, 1<