From 5b43a84a3be25519cc2ce0fb2f68c6f46552aa42 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Feb 2021 17:09:50 -0600 Subject: [PATCH] . --- cmd/sauron/design.org | 199 ++++ cmd/sauron/httputil.go | 274 +++++ cmd/sauron/main.go | 1802 +++++++++++++++++++++++++++++++++ cmd/sauron/populate.sh | 10 + cmd/sauron/sample_schema.json | 1 + cmd/sauron/sauron_test.go | 396 ++++++++ cmd/sauron/schema_test.go | 353 +++++++ cmd/sauron/sqlgen_test.go | 260 +++++ cmd/sauron/test_sqlite.sh | 51 + cmd/sauron/vprint.go | 167 +++ 10 files changed, 3513 insertions(+) create mode 100644 cmd/sauron/design.org create mode 100644 cmd/sauron/httputil.go create mode 100644 cmd/sauron/main.go create mode 100644 cmd/sauron/populate.sh create mode 100644 cmd/sauron/sample_schema.json create mode 100644 cmd/sauron/sauron_test.go create mode 100644 cmd/sauron/schema_test.go create mode 100644 cmd/sauron/sqlgen_test.go create mode 100755 cmd/sauron/test_sqlite.sh create mode 100644 cmd/sauron/vprint.go diff --git a/cmd/sauron/design.org b/cmd/sauron/design.org new file mode 100644 index 000000000..aa579ab34 --- /dev/null +++ b/cmd/sauron/design.org @@ -0,0 +1,199 @@ + +* Oracle Design +- setup server that intersepts PQL queries and proxies to live pilosa server +- implement all the pilosa functions using SQL, specifically the sqlite flavor. This will + provide validation against pilosa results +- provide timings for each query path, PQL and SQL and compare +- this is intended to be a correctness check and not a performance check. the + sql solution is likely to be pretty slow with realitively small numbers ~1millions + #+BEGIN_SRC + .─────. + ,' `. ┌────────────┐ ┌────────┐ +;pilosa json: │ Sauron │ ┌──▶│ pilosa │ +: client ;──────────▶ │ │───┤ └────────┘ + ╲ ╱ │ │ │ ┌────────┐ + `. ,' └────────────┘ └──▶│ sqlite │ + `───' └────────┘ + #+END_SRC +* Setup Sql (sqlite) +#+name: setup test +#+header: :results silent +#+header: :db poc.db + #+BEGIN_SRC sqlite + create table bits (field string,row string, column bigint); + create table bsi (field string,column bigint , val bigint); + CREATE VIEW columns as + select distinct column from bits + UNION + select distinct column from bsi; + + INSERT INTO bits VALUES( 'color','red',1); + INSERT INTO bits VALUES( 'color','red',2); + INSERT INTO bits VALUES( 'color','red',3); + INSERT INTO bits VALUES( 'color','red',4); + INSERT INTO bits VALUES( 'color','red',5); + INSERT INTO bits VALUES( 'color','green',2); + INSERT INTO bits VALUES( 'color','green',4); + INSERT INTO bits VALUES( 'color','green',6); + INSERT INTO bits VALUES( 'color','green',8); + INSERT INTO bits VALUES( 'color','yellow',1); + INSERT INTO bits VALUES( 'color','yellow',3); + INSERT INTO bits VALUES( 'color','yellow',5); + INSERT INTO bits VALUES( 'color','yellow',7); + INSERT INTO bits VALUES( 'color','orange',5); + INSERT INTO bits VALUES( 'color','orange',6); + INSERT INTO bits VALUES( 'color','orange',7); + INSERT INTO bits VALUES( 'color','orange',8); + + INSERT INTO bsi VALUES( 'luminosity',1, 1); + INSERT INTO bsi VALUES( 'luminosity',2, 2); + INSERT INTO bsi VALUES( 'luminosity',3, 4); + INSERT INTO bsi VALUES( 'luminosity',4, 5); + INSERT INTO bsi VALUES( 'luminosity',5, 4); + INSERT INTO bsi VALUES( 'luminosity',6, 3); + INSERT INTO bsi VALUES( 'luminosity',7, 2); + INSERT INTO bsi VALUES( 'luminosity',8, 1); + #+END_SRC + +* TODO PQL to SQL TODO [10/11] +- [X] Row(color=red) + #+begin_src go +(*pql.Query)(&pql.Query{ + Calls: ([]*pql.Call)([]*pql.Call{ + (*pql.Call)(&pql.Call{ + Name: (string)("Row"), + Args: (map[string]interface{})(map[string]interface{}{ + (string)("color"): (string)("red"), + }), + Children: ([]*pql.Call)(nil), + Type: (pql.CallType)(0), + Precomputed: (map[uint64]interface{})(nil), + }), + }), + callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), + conditional: ([]string)(nil), +}) + #+end_src + #+BEGIN_SRC sqlite + select column from bits where field="color" AND row="red" + #+END_SRC +- [X] Count(Row(color=red)) + #+begin_src go +(*pql.Query)(&pql.Query{ + Calls: ([]*pql.Call)([]*pql.Call{ + (*pql.Call)(&pql.Call{ + Name: (string)("Count"), + Args: (map[string]interface{})(nil), + Children: ([]*pql.Call)([]*pql.Call{ + (*pql.Call)(&pql.Call{ + Name: (string)("Row"), + Args: (map[string]interface{})(map[string]interface{}{ + (string)("color"): (string)("red"), + }), + Children: ([]*pql.Call)(nil), + Type: (pql.CallType)(0), + Precomputed: (map[uint64]interface{})(nil), + }), + }), + Type: (pql.CallType)(0), + Precomputed: (map[uint64]interface{})(nil), + }), + }), + callStack: ([]*pql.callStackElem)([]*pql.callStackElem{}), + conditional: ([]string)(nil), +}) + + #+end_src + #+BEGIN_SRC sql + select count(*) from( + select column from bits where field="color" AND row="red" + ) + #+END_SRC +- [X] Intersect(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from bits where field="color" AND row="red" + intersect + select column from bits where field="color" AND row="green" + #+END_SRC +- [X] Union(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from bits where field="color" AND row="red" + union + select column from bits where field="color" AND row="green" + #+END_SRC +- [X] Difference(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from bits where field="color" AND row="red" + except + select column from bits where field="color" AND row="green" + #+END_SRC +- [X] Not(Row(color=red)) + #+BEGIN_SRC sql + select column from columns + except + select column from bits where field="color" AND row="red" + #+END_SRC +- [X] Xor(Row(color=red),Row(color=green)) + #+BEGIN_SRC sql + select column from ( + select column from bits where field="color" AND row="green" + union + select column from bits where field="color" AND row="red" + ) + except + select column from ( + select column from bits where field="color" AND row="green" + intersect + select column from bits where field="color" AND row="red" + ) + #+END_SRC +- [X] Distinct(field=luminosity) + #+BEGIN_SRC sql + select distinct val from bsi where field="lumin" + #+END_SRC +- [X] Distinct(Row(color="red"),field=luminosity) + #+BEGIN_SRC sql + select distinct val from bsi + where field="luminosity" ANDcolumn in ( + select column from bits where row="color" AND row="red" + ) + #+END_SRC +- [X] Distinct(Intersect(Row(color="red"),Row(color="green")),luminosity) + #+BEGIN_SRC sql + select distinct val from bsi + where column in ( + select column from bits where field="color" AND row="red" + intersect + select column from bits where field="color" AND row="green" + ) + AND field="luminosity" + #+END_SRC + +- [X] Count(Union(Intersect(Row(color=red),Row(color=green)),Intersect(Row(color=yellow),Row(color=orange)))) +#+begin_src sqlite +select count(*) from ( +select column from ( + select column from bits where field="color" AND row="green" + intersect + select column from bits where field="color" AND row="red" +) +union +select column from ( + select column from bits where field="color" AND row="yellow" + intersect + select column from bits where field="color" AND row="orange" +)) + +- [ ] Rows(color) +#+begin_src sqlite +select distinct row from bits where field="color" +#+end_src + +- [ ] GroupBy(Rows(color)) + #+BEGIN_SRC sql + select row,count(*) from bits where field="color" group by row; + #+END_SRC +- [ ] GroupBy(Rows(color), Rows(other), limit=7) + #+BEGIN_SRC sql + select row,count(*) from bits where field="color" group by row; + #+END_SRC diff --git a/cmd/sauron/httputil.go b/cmd/sauron/httputil.go new file mode 100644 index 000000000..7056d40e8 --- /dev/null +++ b/cmd/sauron/httputil.go @@ -0,0 +1,274 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "log" + "mime" + "net/http" + "strings" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/pql" + "github.com/pkg/errors" +) + +// validHeaderAcceptJSON returns false if one or more Accept +// headers are present, but none of them are "application/json" +// (or any matching wildcard). Otherwise returns true. +func validHeaderAcceptJSON(header http.Header) bool { + return validHeaderAcceptType(header, "application", "json") +} + +func validHeaderAcceptType(header http.Header, typ, subtyp string) bool { + if v, found := header["Accept"]; found { + for _, v := range v { + t, _, err := mime.ParseMediaType(v) + if err != nil { + switch err { + case mime.ErrInvalidMediaParameter: + // This is an optional feature, so we can keep going anyway. + default: + continue + } + } + spl := strings.SplitN(t, "/", 2) + if len(spl) < 2 { + continue + } + switch { + case spl[0] == typ && spl[1] == subtyp: + return true + case spl[0] == "*" && spl[1] == subtyp: + return true + case spl[0] == typ && spl[1] == "*": + return true + case spl[0] == "*" && spl[1] == "*": + return true + } + } + return false + } + return true +} + +// successResponse is a general success/error struct for http responses. +type successResponse struct { + //h *Handler + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + //Error *Error `json:"error,omitempty"` + Error error +} + +// check determines success or failure based on the error. +// It also returns the corresponding http status code. +func (r *successResponse) check(err error) (statusCode int) { + if err == nil { + r.Success = true + return 0 + } + + cause := errors.Cause(err) + + // Determine HTTP status code based on the error type. + switch cause.(type) { + case pilosa.BadRequestError: + statusCode = http.StatusBadRequest + case pilosa.ConflictError: + statusCode = http.StatusConflict + case pilosa.NotFoundError: + statusCode = http.StatusNotFound + default: + statusCode = http.StatusInternalServerError + } + + r.Success = false + r.Error = err // = &Error{Message: err.Error()} + + return statusCode +} + +// write sends a response to the http.ResponseWriter based on the success +// status and the error. +func (r *successResponse) write(w http.ResponseWriter, err error) { + // Apply the error and get the status code. + statusCode := r.check(err) + + // Marshal the json response. + msg, err := json.Marshal(r) + if err != nil { + http.Error(w, string(msg), http.StatusInternalServerError) + return + } + + // Write the response. + if statusCode == 0 { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write(msg) + if err != nil { + log.Printf("error writing response: %v", err) + return + } + _, err = w.Write([]byte("\n")) + if err != nil { + log.Printf("error writing newline after response: %v", err) + return + } + } else { + http.Error(w, string(msg), statusCode) + } +} + +type postFieldRequest struct { + Options fieldOptions `json:"options"` +} + +// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values, +// and used for input validation. +type fieldOptions struct { + Type string `json:"type,omitempty"` + CacheType *string `json:"cacheType,omitempty"` + CacheSize *uint32 `json:"cacheSize,omitempty"` + Min *pql.Decimal `json:"min,omitempty"` + Max *pql.Decimal `json:"max,omitempty"` + Scale *int64 `json:"scale,omitempty"` + TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` + Keys *bool `json:"keys,omitempty"` + NoStandardView bool `json:"noStandardView,omitempty"` + ForeignIndex *string `json:"foreignIndex,omitempty"` +} + +func (o *fieldOptions) ToPilosaFieldOptions() *pilosa.FieldOptions { + + fo := &pilosa.FieldOptions{ + Type: o.Type, + + /* + //Base int64 `json:"base,omitempty"` + //BitDepth uint `json:"bitDepth,omitempty"` + Min: *o.Min, + Max: *o.Max, + Scale: *o.Scale, + NoStandardView: o.NoStandardView, + CacheSize: *o.CacheSize, + CacheType: *o.CacheType, + TimeQuantum: *o.TimeQuantum, + ForeignIndex: *o.ForeignIndex, + */ + } + if o.Keys != nil { + fo.Keys = *o.Keys + } + return fo +} + +func (o *fieldOptions) validate() error { + // Pointers to default values. + defaultCacheType := pilosa.DefaultCacheType + defaultCacheSize := uint32(pilosa.DefaultCacheSize) + + switch o.Type { + case pilosa.FieldTypeSet, "": + // Because FieldTypeSet is the default, its arguments are + // not required. Instead, the defaults are applied whenever + // a value does not exist. + if o.Type == "" { + o.Type = pilosa.FieldTypeSet + } + if o.CacheType == nil { + o.CacheType = &defaultCacheType + } + if o.CacheSize == nil { + o.CacheSize = &defaultCacheSize + } + if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type set")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type set")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) + } + case pilosa.FieldTypeInt: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { + return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) + } + case pilosa.FieldTypeDecimal: + if o.Scale == nil { + return pilosa.NewBadRequestError(errors.New("decimal field requires a scale argument")) + } else if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { + return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) + } + case pilosa.FieldTypeTime: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time")) + } else if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type time")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type time")) + } else if o.TimeQuantum == nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time")) + } + case pilosa.FieldTypeMutex: + if o.CacheType == nil { + o.CacheType = &defaultCacheType + } + if o.CacheSize == nil { + o.CacheSize = &defaultCacheSize + } + if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type mutex")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) + } + case pilosa.FieldTypeBool: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) + } else if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) + } else if o.Keys != nil { + return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool")) + } else if o.ForeignIndex != nil { + return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key")) + } + default: + return errors.Errorf("invalid field type: %s", o.Type) + } + return nil +} diff --git a/cmd/sauron/main.go b/cmd/sauron/main.go new file mode 100644 index 000000000..497e5346b --- /dev/null +++ b/cmd/sauron/main.go @@ -0,0 +1,1802 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + //"io" + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "os" + "os/exec" + "sort" + "strconv" + "strings" + "time" + + "modernc.org/sqlite" + _ "modernc.org/sqlite" + + //"github.com/gorilla/handlers" + "github.com/gorilla/mux" + "github.com/pilosa/pilosa/v2" + + "github.com/pilosa/pilosa/v2/encoding/proto" + "github.com/pilosa/pilosa/v2/pql" + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pkg/errors" + "github.com/shurcooL/go-goon" +) + +func main() { + +} + +// Sauron sees everything. +// And he converts PQL into SQL. +// +// The PQL is applied to a Pilosa and the SQL +// is applied to a SQL database. Then the +// query results are compared as they are +// returned to the client. +// +type Sauron struct { + cfg *SauronConfig + + sql *PQLToSQL + url string // "https://..." + addr string // host:port + mux *http.ServeMux + srv *http.Server + cmd *exec.Cmd + + // target of proxy + pilosaHttpURL string + + Handler http.Handler +} + +type SauronConfig struct { + Bind string // host:port + BindGRPC string // host:port + GossipPort string // port + DataDir string + + DatabaseName string // for sql database +} + +// IndexInfo2 and FieldInfo2 let us look up +// the schema details quickly by using maps +// to index the fields in an index. +type IndexInfo2 struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options pilosa.IndexOptions `json:"options"` + Fields map[string]*FieldInfo2 `json:"fields"` + ShardWidth uint64 `json:"shardWidth"` +} + +type FieldInfo2 struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options pilosa.FieldOptions `json:"options"` + Views []*pilosa.ViewInfo `json:"views,omitempty"` + indexInfo *IndexInfo2 +} + +func (fi *FieldInfo2) GetIndexName() string { + return fi.indexInfo.Name +} + +func (fi *FieldInfo2) GetIndexCreatedAt() int64 { + return fi.indexInfo.CreatedAt +} + +func (fi *FieldInfo2) GetCreatedAt() int64 { + return fi.CreatedAt +} + +func (fi *FieldInfo2) GetName() string { + return fi.Name +} + +type PQLToSQL struct { + schema pilosa.Schema + DB *sql.DB + + dbprefix string // prefixed to all tables + + // map from indexName to *IndexInfo2 + i2f map[string]*IndexInfo2 + ShardWidth uint64 +} + +func NewPQLToSQL(dbprefix string) *PQLToSQL { + return &PQLToSQL{ + dbprefix: strings.ToLower(dbprefix), // some SQL are not case sensitive, just use lower case. + i2f: make(map[string]*IndexInfo2), + ShardWidth: 1048576, + } +} + +func (ps *PQLToSQL) Start() error { + db, err := sql.Open("sqlite", fmt.Sprintf("%v.db", ps.dbprefix)) + if err != nil { + log.Fatal(err) + } + ps.DB = db + return nil +} +func (ps *PQLToSQL) MustRemove() { + name := fmt.Sprintf("%v.db", ps.dbprefix) + os.Remove(name) + // panicOn(err) +} + +func (ps *PQLToSQL) Stop() error { + return ps.DB.Close() +} +func (ps *PQLToSQL) Store(index, field string, rowIdOrKey interface{}, rowCall []*pql.Call) (bool, error) { + rowSql, _, _, err := ToSql(rowCall, index) + panicOn(err) + insertSql := fmt.Sprintf(`insert into %vλbits select "%v" as field, "%v" as row, column,0 as timestamp from ( %v )`, index, field, rowIdOrKey, rowSql) + res, err := ps.DB.Exec(insertSql) + if err != nil { + return false, err + } + aff, err := res.RowsAffected() + if err != nil { + return false, err + } + return aff > 0, nil +} + +// convert '-' to 'Θ' since db cannot have '-' in table names, +// but pilosa allows it in index and field names. +func dash2theta(s string) (r string) { + return strings.Replace(s, "-", "Θ", -1) +} + +func theta2dash(s string) (r string) { + return strings.Replace(s, "Θ", "-", -1) +} +func (ps *PQLToSQL) CreateSchema(jsonBytes []byte) (err error) { + err = json.Unmarshal(jsonBytes, &ps.schema) + if err != nil { + panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) + } + for _, i := range ps.schema.Indexes { + ps.createDatabase(i.Name) + for _, f := range i.Fields { + panicOn(ps.CreateField(i, f)) + } + } + return nil +} + +func (ps *PQLToSQL) CreateFieldWithOptions(indexName, fieldName string, o fieldOptions) error { + + ii2, ok := ps.i2f[indexName] + if !ok { + return fmt.Errorf("index '%v' not found", indexName) + } + + f2, ok := ii2.Fields[fieldName] + if !ok { + f2 = &FieldInfo2{ + Name: fieldName, + //CreatedAt: f.CreatedAt, + Options: *(o.ToPilosaFieldOptions()), + //Views: f.Views, + indexInfo: ii2, + } + ii2.Fields[fieldName] = f2 + } + return nil +} + +func (ps *PQLToSQL) CreateIndex(indexName string, opts *pilosa.IndexOptions) (err error) { + ii2, ok := ps.i2f[indexName] + if !ok { + ii2 = &IndexInfo2{ + Name: indexName, + //CreatedAt: i.CreatedAt, + Options: *opts, + Fields: make(map[string]*FieldInfo2), + ShardWidth: 1048576, + } + //TODO (twg) figure oiut where to get Shardwidht + ps.i2f[indexName] = ii2 + } + return nil +} +func (ps *PQLToSQL) Query(index string, query []*pql.Call) (err error) { + // sql, err := ToSQL(query, index) + //TODO (twg) implement QUery + return nil +} +func (ps *PQLToSQL) CreateField(i *pilosa.IndexInfo, f *pilosa.FieldInfo) (err error) { + ii2, ok := ps.i2f[i.Name] + if !ok { + ii2 = &IndexInfo2{ + Name: i.Name, + CreatedAt: i.CreatedAt, + Options: i.Options, + Fields: make(map[string]*FieldInfo2), + ShardWidth: i.ShardWidth, + } + ps.i2f[i.Name] = ii2 + } + + f2, ok := ii2.Fields[f.Name] + if !ok { + f2 = &FieldInfo2{ + Name: f.Name, + CreatedAt: f.CreatedAt, + Options: f.Options, + Views: f.Views, + indexInfo: ii2, + } + ii2.Fields[f.Name] = f2 + } + return nil +} + +func (ps *PQLToSQL) createDatabase(index string) (err error) { + idx_prefix := dash2theta(index) + "λ" + // Create fields that don't exist. + sql := fmt.Sprintf("create table %vbits (field string,row,column,timestamp, unique(field,row,column,timestamp) )", idx_prefix) + _, err = ps.DB.Exec(sql) + panicOn(err) + sql = fmt.Sprintf("create table %vbsi (field,column, val bigint,unique(field,column)) ", idx_prefix) + _, err = ps.DB.Exec(sql) + panicOn(err) + sql = fmt.Sprintf(`CREATE VIEW %vcolumns as + select distinct column from %vbits + UNION + select distinct column from %vbsi`, idx_prefix, idx_prefix, idx_prefix) + _, err = ps.DB.Exec(sql) + panicOn(err) + return +} +func (ps *PQLToSQL) GetField(indexName, fieldName string) (*FieldInfo2, error) { + idx, ok := ps.i2f[indexName] + if !ok { + return nil, errors.New(fmt.Sprintf("index not found '%v'", indexName)) + } + f, ok := idx.Fields[fieldName] + if !ok { + return nil, errors.New(fmt.Sprintf("field not found '%v' in index '%v'", fieldName, indexName)) + } + return f, nil +} + +//TODO (twg) below will be moved to its own file later just have it here for quick navigation +type PqlFunction uint8 + +//debate between this +type ResultType uint8 + +const ( + UNKNOWN = ResultType(iota) + BITMAP + INT + ARRAY + GROUPCOUNT + SIGNEDROW + BOOL + PAIRSFIELD + TABLE + AGGSQL +) + +func getAggregateSql(call *pql.Call, index string) (string, error) { + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", errors.New("no _field present") + } + } + var sq string + if len(call.Children) > 0 { + //only child should be a bitmap call + var err error + sq, _, _, err = ToSql(call.Children, index) + if err != nil { + return sq, err + } + sq = " and column in (" + sq + ")" + + } + sqb := fmt.Sprintf(`select %v(val) from %vλbsi where field="%v"%v`, strings.ToLower(call.Name), index, fieldName, sq) + sql := fmt.Sprintf(`select val,count(*) from %vλbsi where val=(%v)%v`, index, sqb, sq) + + return sql, nil +} + +// ToSql takes a pilosa query(ast) and converts it to sql +// assumption: the pql has successfully executed on server +func applyCondition(field string, cond *pql.Condition) (sql string) { + + switch cond.Op { + case pql.EQ, pql.NEQ, pql.LT, pql.LTE, pql.GT, pql.GTE: + val, ok := cond.Uint64Value() + if !ok { + panic("no value for condition") + } + if cond.Op == pql.EQ { + sql = fmt.Sprintf(`field="%v" and val=%d`, field, val) + } else if cond.Op == pql.NEQ { + sql = fmt.Sprintf(`field="%v" and val!=%d`, field, val) + } else if cond.Op == pql.LT { + sql = fmt.Sprintf(`field="%v" and val<%d`, field, val) + } else if cond.Op == pql.LTE { + sql = fmt.Sprintf(`field="%v" and val<=%d`, field, val) + } else if cond.Op == pql.GT { + sql = fmt.Sprintf(`field="%v" and val>%d`, field, val) + } else if cond.Op == pql.GTE { + sql = fmt.Sprintf(`field="%v" and val>=%d`, field, val) + } + case pql.BETWEEN, pql.BTWN_LT_LTE, pql.BTWN_LTE_LT, pql.BTWN_LT_LT: + val, ok := cond.Uint64SliceValue() + if !ok { + panic("bad value for condition") + } + if cond.Op == pql.BETWEEN { + sql = fmt.Sprintf(`field="%v" and %v>=%d and %v<=%d`, field, "val", val[0], "val", val[1]) + } else if cond.Op == pql.BTWN_LT_LTE { + sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[1]) + } else if cond.Op == pql.BTWN_LTE_LT { + sql = fmt.Sprintf(`field="%v" and %v>%d and %v<=%d`, field, "val", val[0], "val", val[0]) + } else if cond.Op == pql.BTWN_LT_LT { + sql = fmt.Sprintf(`field="%v" and %v>%d and %v<%d`, field, "val", val[0], "val", val[1]) + } + } + return +} +func FetchFields(rowsCalls []*pql.Call) ([]string, []interface{}, error) { + //TODO (twg) date fields in rows? + results := make([]string, len(rowsCalls)) + columns := make([]interface{}, 0) + for i, call := range rowsCalls { + //TODO (twg) Time Fields ie from/to args + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return nil, nil, errors.New("no field for rows") + } + } + results[i] = fieldName.(string) + col, ok := call.Args["column"] + if ok { + s := fmt.Sprintf("f%v.column='%v'", i+1, col) + columns = append(columns, s) + } + } + return results, columns, nil +} + +func ToSql(ast []*pql.Call, index string) (string, ResultType, []interface{}, error) { + resultType := UNKNOWN + optional := make([]interface{}, 0) + if len(ast) == 0 { + return "", resultType, optional, nil + } + stack := make([]string, 0) + for _, call := range ast { + switch c := strings.ToLower(call.Name); c { + case "row": + var sql string + if call.HasConditionArg() { + for k, v := range call.Args { + csql := applyCondition(k, v.(*pql.Condition)) + sql = fmt.Sprintf("select distinct column from %vλbsi where %v", index, csql) + + break + } + } else { + field, err := call.FieldArg() + panicOn(err) + row := call.Args[field] + + t, ok := call.Args["from"] + var timestampClause string + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause = fmt.Sprintf(` AND timestamp >= "%v"`, clean) + } + t, ok = call.Args["to"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause += fmt.Sprintf(` AND timestamp < "%v"`, clean) + + } + + sql = fmt.Sprintf(`select distinct column from %vλbits where field="%v" AND row="%v"%v`, index, field, row, timestampClause) + + } + stack = append(stack, sql) + resultType = BITMAP + case "count": + //should have args and should be top level + if len(call.Children) == 0 { + return "", resultType, optional, errors.New("invalid args for count") + } + nestedSQL, _, _, err := ToSql(call.Children, index) + panicOn(err) + sql := "select count(*) from( " + nestedSQL + " )" + stack = append(stack, sql) + resultType = INT + case "union", "intersect": + var sql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(sql) == 0 { + sql = sq + } else { + sql += " " + c + " " + sq + } + } + sql = "select column from(" + sql + ")" + stack = append(stack, sql) + resultType = BITMAP + case "difference": + var sql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(sql) == 0 { + sql = sq + } else { + sql += " except " + sq + } + } + sql = "select column from(" + sql + ")" + stack = append(stack, sql) + resultType = BITMAP + case "not": + //should have args and should be top level + if len(call.Children) == 0 { + return "", resultType, optional, errors.New("invalid args for count") + } + nestedSQL, _, _, err := ToSql(call.Children, index) + panicOn(err) + sql := fmt.Sprintf("select column from %vλcolumns except %v", index, nestedSQL) + stack = append(stack, sql) + resultType = BITMAP + case "xor": + var unionSql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(unionSql) == 0 { + unionSql = sq + } else { + unionSql += " union " + sq + } + } + unionSql = "select column from(" + unionSql + ")" + var intersectSql string + for _, subcall := range call.Children { + sq, _, _, err := ToSql([]*pql.Call{subcall}, index) + panicOn(err) + if len(intersectSql) == 0 { + intersectSql = sq + } else { + intersectSql += " intersect " + sq + } + } + intersectSql = "select column from(" + intersectSql + ")" + sql := unionSql + " except " + intersectSql + stack = append(stack, sql) + resultType = BITMAP + + case "distinct": + //TODO validate field type, only valid for bsi fields + fieldName, ok := call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("field not specified on distinct call") + } + var sql string + if len(call.Children) > 1 { + return "", resultType, optional, errors.New(fmt.Sprintf("unexpected args to distinct query:'%v'", call.Children)) + } + sql = fmt.Sprintf(`select distinct val from %vλbsi where field="%v"`, index, fieldName) + if len(call.Children) > 0 { + sq, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) + panicOn(err) + sql += " AND column in (" + sq + ")" + } + stack = append(stack, sql) + resultType = SIGNEDROW + case "rows": + //TODO (twg) Time Fields ie from/to args + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("no _field present") + } + } + optional = append(optional, fieldName) + sql := fmt.Sprintf(`select distinct field, row from %vλbits where field="%v"`, index, fieldName) + col, ok := call.Args["column"] + if ok { + sql = fmt.Sprintf("%v and column='%v'", sql, col) + } + prev, ok := call.Args["previous"] + if ok { + sql = fmt.Sprintf("%v and row>'%v'", sql, prev) + } + t, ok := call.Args["from"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) + sql += timestampClause + } + t, ok = call.Args["to"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) + sql += timestampClause + } + sql += " order by row" + limit, ok := call.Args["limit"] + if ok { + sql = fmt.Sprintf("%v limit %v", sql, limit) + } + stack = append(stack, sql) + resultType = ARRAY + case "groupby": + //need to figure out how many fields we will be looking at + fields, optColumns, err := FetchFields(call.Children) + panicOn(err) + sql := "select " + for i := range fields { + f := fmt.Sprintf("f%d", i+1) + if i == 0 { + sql += fmt.Sprintf(" %s.field, %s.row", f, f) + } else { + + sql += fmt.Sprintf(",%s.field, %s.row", f, f) + } + } + tname := fmt.Sprintf("%vλbits", index) + sql += fmt.Sprintf(",count(*) as cnt from %v f1", tname) + if len(fields) > 1 { + for i := range fields[1:] { + f := fmt.Sprintf("f%d", i+2) + sql += fmt.Sprintf(" inner join %v %v on f1.column = %v.column", tname, f, f) + } + } + sql += " where " + gb := "" + for i, field := range fields { + f := fmt.Sprintf("f%d", i+1) + if i == 0 { + sql += fmt.Sprintf("%v.field='%v'", f, field) + gb += fmt.Sprintf("%v.field,%v.row", f, f) + } else { + sql += fmt.Sprintf("and %v.field='%v'", f, field) + gb += fmt.Sprintf(",%v.field,%v.row", f, f) + } + } + if len(optColumns) > 0 { + for _, column := range optColumns { + sql += fmt.Sprintf(" and %v ", column) + } + } + sql += " group by " + gb + " order by cnt desc" + + optional = append(optional, len(fields)) + + stack = append(stack, sql) + resultType = GROUPCOUNT + case "topn": + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("no _field present") + } + } + var sq string + if len(call.Children) > 0 { + if len(call.Children) > 1 { + return "", resultType, optional, errors.New("invalid topn") + } + var err error + sq, _, _, err = ToSql(call.Children, index) + panicOn(err) + sq = " and column in (" + sq + ")" + + } + sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) + + limit, ok := call.Args["n"] + if ok { + sql += fmt.Sprintf(" limit %v", limit) + } + stack = append(stack, sql) + resultType = PAIRSFIELD + case "topk": + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + return "", resultType, optional, errors.New("no _field present") + } + } + var sq string + filter, ok := call.Args["filter"] + if ok { + //assuming this works in pilosa so the filter must be a correct bitmap call + //ie row/union/intersect/xor/difference + var err error + sq, _, _, err = ToSql([]*pql.Call{filter.(*pql.Call)}, index) + panicOn(err) + sq = " and column in (" + sq + ")" + + } + t, ok := call.Args["from"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp >= "%v"`, clean) + sq += timestampClause + } + t, ok = call.Args["to"] + if ok { + clean := strings.Replace(t.(string), "T", " ", 1) + timestampClause := fmt.Sprintf(` AND timestamp < "%v"`, clean) + sq += timestampClause + } + sql := fmt.Sprintf(`select row,count(*) as cnt from %vλbits where field="%v"%s group by row order by cnt desc`, index, fieldName, sq) + + limit, ok := call.Args["k"] + if ok { + sql += fmt.Sprintf(" limit %v", limit) + } + stack = append(stack, sql) + resultType = PAIRSFIELD + case "min", "max": + sql, err := getAggregateSql(call, index) + panicOn(err) + stack = append(stack, sql) + resultType = AGGSQL + case "sum": + fieldName, ok := call.Args["_field"] + if !ok { + fieldName, ok = call.Args["field"] + if !ok { + panic("no _field present") + } + } + var sq string + if len(call.Children) > 0 { + //only child should be a bitmap call + var err error + sq, _, _, err = ToSql(call.Children, index) + panicOn(err) + sq = " and column in (" + sq + ")" + + } + sql := fmt.Sprintf(`select sum(val),count(*) from %vλbsi where field="%v"%v`, index, fieldName, sq) + stack = append(stack, sql) + resultType = AGGSQL + case "constrow": + sql := "select column1 as column from (values " + columns, ok := call.Args["columns"].([]interface{}) + + if ok { + for i, col := range columns { + if i != 0 { + sql += "," + } + sql += fmt.Sprintf("(%v)", col) + + } + sql += ")" + + } + + stack = append(stack, sql) + resultType = BITMAP + case "all": + sql := fmt.Sprintf("select column from %vλcolumns", index) + stack = append(stack, sql) + resultType = BITMAP + case "extract": + // extract is a table response + bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) + panicOn(err) + var rowsPart string + if len(call.Children) > 1 { + rows := call.Children[1:] + rowsPart = "select field,row from(" + for i, row := range rows { + part, _, _, err1 := ToSql([]*pql.Call{row}, index) + panicOn(err1) + if i > 0 { + rowsPart += "union all select field,row from (" + } + rowsPart += part + ")" + } + } + var sql string + if len(rowsPart) == 0 { + sql = "select '' as field, '' as row,column from (" + bm + ")" + } else { + sql = "select sq.field, sq.row, b.column from (" + sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) + sql += " where b.column in (" + bm + ")" + } + + stack = append(stack, sql) + resultType = TABLE + case "unionrows": + var rowsPart string + rowsPart = "select field,row from(" + for i, row := range call.Children { + part, _, _, err1 := ToSql([]*pql.Call{row}, index) + panicOn(err1) + if i > 0 { + rowsPart += "union all select field,row from (" + } + rowsPart += part + ")" + } + sql := "select distinct b.column from (" + sql += rowsPart + fmt.Sprintf(") sq inner join %vλbits b on sq.field = b.field and sq.row = b.row", index) + stack = append(stack, sql) + resultType = BITMAP + case "includescolumn": + bm, _, _, err := ToSql([]*pql.Call{call.Children[0]}, index) + panicOn(err) + col, ok := call.Args["column"] + if !ok { + return "", resultType, optional, errors.New("missing column parameter") + } + + sql := fmt.Sprintf("select count(*)>0 from (select column from(%v) where column='%v')", bm, col) + stack = append(stack, sql) + resultType = BOOL + default: + goon.Dump(call) + return "", resultType, optional, errors.New(fmt.Sprintf("unknown call '%v'", c)) + } + } + if len(stack) == 0 { + //not sure if i should error on an NOP + return "", resultType, optional, nil + } + return stack[0], resultType, optional, nil +} + +func NewSauron(cfg *SauronConfig) *Sauron { + + addr := "127.0.0.1:8080" + url := "http://" + addr + s := &Sauron{ + cfg: cfg, + addr: addr, + url: url, + sql: NewPQLToSQL(cfg.DatabaseName), + srv: &http.Server{ + Addr: addr, + ReadTimeout: 50 * time.Millisecond, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + }, + } + s.Handler = newRouter(s) + s.srv.Handler = s.Handler + return s +} + +func (p *Sauron) Start() (urlstring string, err error) { + started := make(chan struct{}) + + panicOn(p.sql.Start()) + + // start a pilosa server + bind := p.cfg.Bind + dataDir := p.cfg.DataDir + bindGRPC := p.cfg.BindGRPC + gossipPort := p.cfg.GossipPort + + p.cmd = exec.Command("pilosa", "server", + "--bind", bind, + "--data-dir", dataDir, + "--bind-grpc", bindGRPC, + "--gossip.port", gossipPort) + err = p.cmd.Start() + panicOn(err) + + // wait for it to be listening + p.pilosaHttpURL = "http://" + bind + ok := false + seconds := 20 + for i := 0; i < seconds; i++ { + nc, err := net.Dial("tcp", bind) + if err == nil { + nc.Close() + ok = true + break + } + time.Sleep(time.Second) + } + if !ok { + return "", fmt.Errorf("could not contact pilosa server on %v after %v seconds. pid = %v", bind, seconds, p.cmd.Process.Pid) + } + go func() { + close(started) + + err = p.srv.ListenAndServe() + if err != nil { + if strings.Contains(err.Error(), "Server closed") { + return + } + } + panicOn(err) + }() + <-started + return p.url, nil +} + +func (p *Sauron) Stop() { + panicOn(p.sql.Stop()) + err := p.srv.Shutdown(context.Background()) + panicOn(err) + // Kill pilosa + if err = p.cmd.Process.Kill(); err != nil { + vv("failed to kill process: ", err) + } +} + +// GetAvailPort asks the OS for an unused port. +// There's a race here, where the port could be grabbed by someone else +// before the caller gets to Listen on it, but we are only using +// it to find a random port for the test hang debugging. +// Moreover, in practice such races are rare. Just ask for +// it again if the port is taken. +// Uses net.Listen("tcp", ":0") to determine a free port, then +// releases it back to the OS with Listener.Close(). +func GetAvailPort() int { + l, _ := net.Listen("tcp", ":0") + r := l.Addr() + l.Close() + return r.(*net.TCPAddr).Port +} + +// newRouter creates a new mux http router. +func newRouter(handler *Sauron) http.Handler { + router := mux.NewRouter() + + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") + router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") + router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") + + router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex") + router.HandleFunc("/index/{index}/field/", handler.handlePostField).Methods("POST").Name("PostField") + router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField") + router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring") + router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport") + + return router +} + +// handleGetSchema handles GET /schema requests. +func (s *Sauron) handleGetSchema(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + _ = body + _ = resp + _ = err + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(resp.StatusCode) + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + _, err = w.Write(respBody) + panicOn(err) + + // B) there is no SQL version of get schema. + +} + +// creates an entire schema from the dumped schema. +// requires completed structures, no defaults applied +func (s *Sauron) handlePostSchema(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + + w.WriteHeader(resp.StatusCode) + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + _, err = w.Write(respBody) + panicOn(err) + + // B) SQL + panicOn(s.sql.CreateSchema(body)) +} + +// handlePostField handles /index/{index}/field/{field} requests +func (s *Sauron) handlePostIndex(w http.ResponseWriter, r *http.Request) { + body, resp, err := s.ForwardHandler(w, r) + _ = body + _ = resp + _ = err + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + indexName, ok := mux.Vars(r)["index"] + if !ok { + http.Error(w, "index name is required", http.StatusBadRequest) + return + } + req := struct { + Options pilosa.IndexOptions + }{ + Options: pilosa.IndexOptions{ + Keys: false, + TrackExistence: true, + }} + if len(body) > 0 { //only process if options are present + err = json.Unmarshal(body, &req) + panicOn(err) + } + err = s.sql.CreateIndex(indexName, &req.Options) + panicOn(err) + w.Write(respBody) +} + +// handlePostField handles /index/{index}/field/{field} requests +func (s *Sauron) handlePostField(w http.ResponseWriter, r *http.Request) { + body, resp, err := s.ForwardHandler(w, r) + _ = body + _ = resp + _ = err + indexName, ok := mux.Vars(r)["index"] + if !ok { + http.Error(w, "index name is required", http.StatusBadRequest) + return + } + + fieldName, ok := mux.Vars(r)["field"] + if !ok { + http.Error(w, "field name is required", http.StatusBadRequest) + return + } + + respBody, err := ioutil.ReadAll(resp.Body) + panicOn(err) + // Decode request. + var req postFieldRequest + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + err = dec.Decode(&req) + if err != io.EOF { + panicOn(err) + } + err = s.sql.CreateFieldWithOptions(indexName, fieldName, req.Options) + _, err = w.Write(respBody) + panicOn(err) +} + +// handlePostQuery handles /index/{index}/query requests +func (s *Sauron) handlePostQuery(w http.ResponseWriter, req *http.Request) { + + body, resp, err := s.ForwardHandler(w, req) + pilosaJSON, err := ioutil.ReadAll(resp.Body) + panicOn(err) + var pilosaResponse pilosa.QueryResponse // DEBUG + dec := json.NewDecoder(bytes.NewReader(pilosaJSON)) + err = dec.Decode(&pilosaResponse) + panicOn(err) + qreq := &pilosa.QueryRequest{ + Index: mux.Vars(req)["index"], + Query: string(body), + } + + oracleResponse, err := s.sql.IssueQuery(qreq) + + if !pilosaResponse.Equals(oracleResponse) { + //plan is to write out a useful query response indicating a discrepancy + //TODO (twg) write diff in response somehow + vv("oracle:") + prettyJson(*oracleResponse) + vv("pilosa:") + prettyJson(pilosaResponse) + panic("barf") + } +} +func prettyJson(a pilosa.QueryResponse) { + src, _ := a.MarshalJSON() + dst := &bytes.Buffer{} + if err := json.Indent(dst, src, "", " "); err != nil { + panic(err) + } + vv("%v", string(dst.Bytes())) +} + +func (s *Sauron) FieldInfo(index, field string) (*FieldInfo2, error) { + idx, ok := s.sql.i2f[index] + if ok { + f, ok := idx.Fields[field] + if ok { + return f, nil + } + return nil, errors.New(fmt.Sprintf("field not in schema '%v'", field)) + } + return nil, errors.New(fmt.Sprintf("index not in schema '%v'", index)) +} +func (s *Sauron) handlePostImport(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + _ = body + _ = resp + _ = err + // Get index and field type to determine how to handle the + // import data. + indexName := mux.Vars(req)["index"] + fieldName := mux.Vars(req)["field"] + + _ = indexName + _ = fieldName + + // If the clear flag is true, treat the import as clear bits. + q := req.URL.Query() + doClear := q.Get("clear") == "true" + doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" + _ = doClear + _ = doIgnoreKeyCheck + + info, err := s.FieldInfo(indexName, fieldName) + panicOn(err) + // Unmarshal request based on field type. + fieldType := info.Options.Type + if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal { + iv := &pilosa.ImportValueRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, iv); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + //TODO (twg) ignore columnKeys at the moment + //do importValue + for i, column := range iv.ColumnIDs { + if len(iv.Values) > 0 { + s.sql.Set(indexName, column, fieldName, iv.Values[i], "") + } else if len(iv.FloatValues) > 0 { //TODO (twg) these may not be reachable + s.sql.Set(indexName, column, fieldName, iv.FloatValues[i], "") //probably some issues here + } else if len(iv.StringValues) > 0 { + s.sql.Set(indexName, column, fieldName, iv.StringValues[i], "") + } + } + } else { + // Field type: set, time, mutex + // Marshal into request object. + ir := &pilosa.ImportRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, ir); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + //do import + action := s.sql.Set + if ir.Clear { + action = func(index string, column uint64, field string, value interface{}, ts string) (bool, error) { + return s.sql.Clear(index, column, field, value) + } + } + changed := false + _ = changed + for i, column := range ir.ColumnIDs { + ts := "" + if len(ir.Timestamps) > 0 { + t := time.Unix(ir.Timestamps[i], 0) + ts = t.Format("2006-01-02 15:04:05") + } + if len(ir.RowIDs) > 0 { + change, err := action(indexName, column, fieldName, ir.RowIDs[i], ts) + panicOn(err) + if change { + changed = true + } + } else if len(ir.RowKeys) > 0 { //TODO (twg) these may not be reachable + change, err := action(indexName, column, fieldName, ir.RowKeys[i], ts) + panicOn(err) + if change { + changed = true + } + } + } + } +} +func (s *Sauron) handlePostImportRoaring(w http.ResponseWriter, req *http.Request) { + body, resp, err := s.ForwardHandler(w, req) + _ = body + _ = resp + _ = err + // Get index and field type to determine how to handle the + // import data. + indexName := mux.Vars(req)["index"] + fieldName := mux.Vars(req)["field"] + + irr := &pilosa.ImportRoaringRequest{} + err = proto.DefaultSerializer.Unmarshal(body, irr) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + //do i need the shard? + urlVars := mux.Vars(req) + shard, err := strconv.ParseUint(urlVars["shard"], 10, 64) + if err != nil { + panicOn(err) + } + for v, blob := range irr.Views { + _ = v //ignore the view for now + rit, err := roaring.NewRoaringIterator(blob) + panicOn(err) + s.sql.ImportRoaring(indexName, fieldName, shard, rit) + } +} + +//end handler +func (s *Sauron) ForwardHandler(w http.ResponseWriter, req *http.Request) (body []byte, resp *http.Response, err error) { + // TODO (twg) replace all the instances + // we need to buffer the body if we want to read it here and send it + // in the request. + body, err = ioutil.ReadAll(req.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // you can reassign the body if you need to parse it as multipart + req.Body = ioutil.NopCloser(bytes.NewReader(body)) + + // create a new url from the raw RequestURI sent by the client + url := fmt.Sprintf("%s%s", s.pilosaHttpURL, req.RequestURI) + + proxyReq, err := http.NewRequest(req.Method, url, bytes.NewReader(body)) + + // We may want to filter some headers, otherwise we could just use a shallow copy + // proxyReq.Header = req.Header + proxyReq.Header = make(http.Header) + for h, val := range req.Header { + proxyReq.Header[h] = val + } + + resp, err = http.DefaultClient.Do(proxyReq) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + return +} + +// ImportRoaring must not be a bsi field..check higher up the stack +func (ps *PQLToSQL) ImportRoaring(index, field string, shard uint64, itr roaring.RoaringIterator) { + rb := roaring.NewBitmap() + for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { + rb.Put(itrKey, synthC) + } + for _, pos := range rb.Slice() { + row, column := pos/ps.ShardWidth, (shard*ps.ShardWidth)+(pos%ps.ShardWidth) + //ps.SetWithInt64Field(index, column, field, int64(row)) + ps.Set(index, column, field, int64(row), "") + } +} + +func (ps *PQLToSQL) isBSI(index, field string) (full string, isBSI bool) { + idx, ok := ps.i2f[index] + if !ok { + panic(fmt.Errorf("isBSI: no such index '%v'", index)) + } + + fld, ok := idx.Fields[field] + if !ok { + panic(fmt.Errorf("isBSI: no such field '%v'", field)) + } + if fld.Options.Type == "int" { + return dash2theta(index) + "λbsi", true + } + return dash2theta(index) + "λbits", false +} + +// Set is the basic bit operation and value operation as well, depending on the field type +func (ps *PQLToSQL) Set(index string, column uint64, field string, value interface{}, timestamp string) (bool, error) { + table, isBSI := ps.isBSI(index, field) + var sql string + if isBSI { + sql = fmt.Sprintf("INSERT INTO %v VALUES( '%v','%v', '%v')", table, field, column, value) + } else { + //if len(timestamp) == 0 { + // sql = fmt.Sprintf("INSERT INTO %v (field,row,column)VALUES( '%v','%v', '%v')", table, field, value, column) + // } else { + //TODO(twg) will need to make the timestamp as granular as the field is configured. + sql = fmt.Sprintf("INSERT INTO %v (field,row,column,timestamp)VALUES( '%v','%v', '%v','%v')", table, field, value, column, timestamp) + // } + } + + _, err := ps.DB.Exec(sql) + if err != nil { + if sqerr, ok := err.(*sqlite.Error); ok { + if sqerr.Code() == 2067 { //constraint error + return false, nil + } + } + } + panicOn(err) + return true, nil +} + +// Clear removes the specified column or value +func (ps *PQLToSQL) Clear(index string, column uint64, field string, value interface{}) (bool, error) { + table, isBSI := ps.isBSI(index, field) + var sql string + if isBSI { + sql = fmt.Sprintf("delete from %v where field='%v' and column='%v'", table, field, column) + } else { + sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' and column='%v'", table, field, value, column) + } + + res, err := ps.DB.Exec(sql) + panicOn(err) + r, err := res.RowsAffected() + panicOn(err) + if r == 0 { + return false, nil + } + return true, nil +} + +// ClearRow removes entire Row(feature) +func (ps *PQLToSQL) ClearRow(index string, field string, row interface{}) (bool, error) { + table, isBSI := ps.isBSI(index, field) + var sql string + if isBSI { + panic("bsi can't clear row allowed") + } else { + sql = fmt.Sprintf("delete from %v where field='%v' and row='%v' ", table, field, row) + } + + r, err := ps.DB.Exec(sql) + panicOn(err) + n, err := r.RowsAffected() + return n > 0, err +} + +func sliceToString(slc []int64) (r string) { + if len(slc) == 0 { + return + } + for _, i := range slc { + r += fmt.Sprintf("%v,", i) + } + return r[:len(r)-1] +} + +const columnLabel = "col" + +func (ps *PQLToSQL) IssueQuery(req *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { + index := req.Index + q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() + if err != nil { + return nil, errors.Wrap(err, "PQLToSQL.IssueQuery parsing PQL") + } + var results pilosa.QueryResponse + for _, c := range q.Calls { + + if callIndex := c.CallIndex(); callIndex != "" { + index = callIndex + } + + // Handle the field arg. + switch strings.ToLower(c.Name) { + //handle the write queries + //TODO (twg) figure out sqlite changes() function to determine number of bits changed + case "set": + // Read colID. + colID, ok, err := c.UintArg("_" + columnLabel) + if err != nil { + return nil, fmt.Errorf("reading Set() column: %v", err) + } else if !ok { + return nil, fmt.Errorf("Set() column argument '%v' required", columnLabel) + } + ts, ok := c.Args["_timestamp"] + var timestamp string + if ok { + timestamp = strings.Replace(ts.(string), "T", " ", 1) + } + + field, err := c.FieldArg() + if err == nil { + rs, err := ps.Set(index, colID, field, c.Args[field], timestamp) + panicOn(err) + results.Results = append(results.Results, rs) + } else { + return nil, err + } + case "clear": + // Read colID. + colID, ok, err := c.UintArg("_" + columnLabel) + if err != nil { + return nil, fmt.Errorf("reading Clear() column: %v", err) + } else if !ok { + return nil, fmt.Errorf("Clear() column argument '%v' required", columnLabel) + } + + field, err := c.FieldArg() + if err == nil { + rs, err := ps.Clear(index, colID, field, c.Args[field]) + if err != nil { + return nil, fmt.Errorf("Clear() column: %v", err) + } + results.Results = append(results.Results, rs) + } else { + vv("c.FieldArg() returned '%v'", err) + } + case "clearrow": + field, err := c.FieldArg() + panicOn(err) + rs, err := ps.ClearRow(index, field, c.Args[field]) + panicOn(err) + results.Results = append(results.Results, rs) + case "store": + field, err := c.FieldArg() + panicOn(err) + rs, err := ps.Store(index, field, c.Args[field], c.Children) + panicOn(err) + results.Results = append(results.Results, rs) + default: //TODO (twg) move ToSQL into main package + rs, err := ps.readQuery(index, q) + panicOn(err) + results.Results = append(results.Results, rs) + } + } + return &results, nil +} +func (ps *PQLToSQL) handleBitmapCall(sql string) (*pilosa.Row, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + defer rows.Close() + columns := make([]uint64, 0) + for rows.Next() { + //really need a way to know what the results look like + var column uint64 + rows.Scan(&column) + columns = append(columns, column) + } + row := pilosa.NewRow(columns...) + return row, nil +} + +func (ps *PQLToSQL) handleIntCall(sql string) (count uint64, err error) { + rows, err := ps.DB.Query(sql) + if err != nil { + return 0, err + } + defer rows.Close() + for rows.Next() { + //really need a way to know what the results look like + rows.Scan(&count) + return + } + return +} + +type Table struct { +} + +func (ps *PQLToSQL) handleSignedRowCall(sql string) (pilosa.SignedRow, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + + var id uint64 + var pos []uint64 + var neg []uint64 + defer rows.Close() + for rows.Next() { + err := rows.Scan(&id) + if err != nil { + return pilosa.SignedRow{}, err + } + if id >= 0 { + pos = append(pos, id) + } else { + neg = append(neg, id) + } + } + err = rows.Err() + if err != nil { + return pilosa.SignedRow{}, err + } + posRow := pilosa.NewRow(pos...) + negRow := pilosa.NewRow(neg...) + return pilosa.SignedRow{Pos: posRow, Neg: negRow}, nil + +} +func (ps *PQLToSQL) handleStringArrayCall(sql string) ([]interface{}, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + rowKeys := make([]interface{}, 0) + var field string + var id string + defer rows.Close() + for rows.Next() { + err := rows.Scan(&field, &id) + if err != nil { + return nil, err + } + rowKeys = append(rowKeys, id) + } + err = rows.Err() + if err != nil { + return nil, err + } + return rowKeys, nil +} + +func (ps *PQLToSQL) handleIntArrayCall(sql string) (map[string][]interface{}, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + rowIDs := make([]interface{}, 0) + var id uint64 + var field string + defer rows.Close() + for rows.Next() { + err := rows.Scan(&field, &id) + if err != nil { + return nil, err + } + rowIDs = append(rowIDs, id) + } + err = rows.Err() + if err != nil { + return nil, err + } + m := map[string][]interface{}{"rows": rowIDs, "keys": []interface{}{}} + return m, nil +} +func (ps *PQLToSQL) handleGroupCountCall(sqlin string, index string, numFields int) (pilosa.SortedGroupCount, error) { + rows, err := ps.DB.Query(sqlin) + panicOn(err) + //groupCounts := make([]pilosa.GroupCount, 0) + var groupCounts pilosa.SortedGroupCount + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + return nil, err + } + row := make([]interface{}, len(cols)) + for i := range cols { + row[i] = new(sql.RawBytes) + } + for rows.Next() { + // need a row*NumFields + cnt + //err := rows.Scan(&field, &row, &count) + groupCount := pilosa.GroupCount{} + + err := rows.Scan(row...) + if err != nil { + return nil, err + } + for i := 0; i < len(row)-1; i += 2 { + ptr := row[i].(*sql.RawBytes) + fieldName := string(*ptr) + fr := pilosa.FieldRow{Field: fieldName} + fi, err := ps.GetField(index, fieldName) + if err != nil { + return nil, err + } + ptr = row[i+1].(*sql.RawBytes) + v := string(*ptr) + if fi.Options.Keys { + fr.RowKey = v + + } else { + iv, err := strconv.Atoi(v) + if err != nil { + return nil, err + } + fr.RowID = uint64(iv) + } + + groupCount.Group = append(groupCount.Group, fr) + + } + ptr := row[len(row)-1].(*sql.RawBytes) + data := string(*ptr) //convert to string + i, err := strconv.ParseInt(data, 10, 64) + panicOn(err) + groupCount.Count = uint64(i) + groupCounts = append(groupCounts, groupCount) + + } + + err = rows.Err() + if err != nil { + return nil, err + } + return groupCounts, nil +} +func (ps *PQLToSQL) handlePairsFieldCall(sql, index string) (pilosa.PairsField, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + var count uint64 + var row string + defer rows.Close() + res := pilosa.PairsField{} + for rows.Next() { + err := rows.Scan(&row, &count) + if err != nil { + return pilosa.PairsField{}, err + } + pair := pilosa.Pair{Key: row, Count: count} + res.Pairs = append(res.Pairs, pair) + } + err = rows.Err() + if err != nil { + return pilosa.PairsField{}, err + } + return res, nil +} + +// special structures to deal with Extract. In the future I would like to use +// pilosa structs, but dealing with keys makes that complicated for now +// note the column is a string, but i only worry about the type when marshalling out for +// the sqlite part is typeless. Handling keys is almost seemless and not requiring translation +type extractColumns struct { + column string + rows map[int][]interface{} + parent *sauronExtractedIDMatrix +} + +func (eid extractColumns) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + buf.WriteString(fmt.Sprintf(`{"column": %v,"rows":[`, eid.column)) + rows := make([][]interface{}, len(eid.parent.fields)) + for k, v := range eid.rows { + rows[k] = v + } + for i, grow := range rows { + if i != 0 { + buf.WriteByte(44) //write the comma + } + isKey := eid.parent.fields[i].isKey + buf.WriteString(`[`) + for d, row := range grow { + if d != 0 { + buf.WriteByte(44) //write the comma + } + if isKey { + buf.WriteString(fmt.Sprintf(`"%v"`, row)) + } else { + buf.WriteString(fmt.Sprintf(`%v`, row)) + } + + } + buf.WriteString(`]`) + } + + buf.WriteString(`]}`) + return buf.Bytes(), nil +} + +type extractFields struct { + Name string `json:"name"` + Type string `json:"type"` + isKey bool +} +type sauronExtractedIDMatrix struct { + columns []extractColumns + fields []extractFields +} + +func (eid sauronExtractedIDMatrix) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + sort.Slice(eid.columns, func(i, j int) bool { + return eid.columns[i].column < eid.columns[j].column + }) + buf.WriteString(`{"columns":[`) + for i, col := range eid.columns { + b, err := col.MarshalJSON() + panicOn(err) + if i != 0 { + buf.WriteByte(44) //write the comma + } + buf.Write(b) + } + + buf.WriteString(`],"fields":[`) + for i, f := range eid.fields { + if i != 0 { + buf.WriteByte(44) + } + b, err := json.Marshal(f) + panicOn(err) + buf.Write(b) + } + buf.WriteString(`]}`) + return buf.Bytes(), nil +} + +func (ps *PQLToSQL) handleTableCall(sql, index string) (sauronExtractedIDMatrix, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + var field string + var row string + var column string + defer rows.Close() + col := make(map[string]map[string][]interface{}) + fidx := make(map[string]int) + results := sauronExtractedIDMatrix{} + counter := 0 + empty := make(map[string][]interface{}) + for rows.Next() { + err := rows.Scan(&field, &row, &column) + if err != nil { + return sauronExtractedIDMatrix{}, err + } + if field != "" { + _, ok := fidx[field] + if !ok { + fidx[field] = counter + counter++ + } + rows, ok := col[column] + if !ok { + rows = make(map[string][]interface{}) + } + rows[field] = append(rows[field], row) + //results.Columns = append(results.Columns, pilosa.ExtractedIDColumn{ColumnID: column, Rows: [][]uint64{}}) + col[column] = rows + } else { //constrow, or no filed info + col[column] = empty + } + } + err = rows.Err() + if err != nil { + return sauronExtractedIDMatrix{}, err + } + for c, rowMap := range col { + idc := extractColumns{rows: make(map[int][]interface{}), parent: &results} + idc.column = c + for fld, rows := range rowMap { + idx := fidx[fld] + idc.rows[idx] = rows + } + results.columns = append(results.columns, idc) + } + results.fields = make([]extractFields, len(fidx)) + for fieldName, idx := range fidx { + fi, _ := ps.GetField(index, fieldName) + // TODO (twg) hack need to figure out proper way to determine type + aType := "[]uint64" + if fi.Options.Keys { + aType = "[]string" + } + results.fields[idx] = extractFields{Name: fieldName, Type: aType, isKey: fi.Options.Keys} + } + + return results, nil +} + +func (ps *PQLToSQL) handleAggSQLCall(sql, index string) (pilosa.ValCount, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + defer rows.Close() + results := pilosa.ValCount{} + var val int64 + var count int64 + for rows.Next() { + err := rows.Scan(&val, &count) + if err != nil { + return pilosa.ValCount{}, err + } + results.Count = count + results.Val = val + break //shouldn't be anymore rows but being safe + } + err = rows.Err() + if err != nil { + return pilosa.ValCount{}, err + } + return results, nil +} +func (ps *PQLToSQL) handleBoolCall(sql, index string) (bool, error) { + rows, err := ps.DB.Query(sql) + panicOn(err) + defer rows.Close() + var val bool + for rows.Next() { + err := rows.Scan(&val) + if err != nil { + return false, err + } + break //shouldn't be anymore rows but being safe + } + err = rows.Err() + if err != nil { + return val, err + } + return val, nil +} + +func (ps *PQLToSQL) readQuery(index string, query *pql.Query) (interface{}, error) { + //Row(color=red) + sql, resultType, optional, err := ToSql(query.Calls, index) + panicOn(err) + switch resultType { + case BITMAP: + return ps.handleBitmapCall(sql) + case INT: + return ps.handleIntCall(sql) + case SIGNEDROW: + return ps.handleSignedRowCall(sql) + case ARRAY: + fieldName := optional[0].(string) + info, ok := ps.i2f[index] + if !ok { + return nil, errors.New(fmt.Sprintf("index not in schema %v", index)) + } + field, ok := info.Fields[fieldName] + if !ok { + return nil, errors.New(fmt.Sprintf("field not in schema %v", fieldName)) + } + if field.Options.Keys { + return ps.handleStringArrayCall(sql) + } else { + return ps.handleIntArrayCall(sql) + } + case GROUPCOUNT: + return ps.handleGroupCountCall(sql, index, optional[0].(int)) + case PAIRSFIELD: + return ps.handlePairsFieldCall(sql, index) + case TABLE: + return ps.handleTableCall(sql, index) + case AGGSQL: + return ps.handleAggSQLCall(sql, index) + case BOOL: + return ps.handleBoolCall(sql, index) + default: + vv("Unknown query Type %v", resultType) + } + return nil, nil +} + +type Int64Slice []int64 + +func (p Int64Slice) Len() int { return len(p) } +func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p Int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +// on tables starting with ps.dbprefix will be returned. +func (ps *PQLToSQL) ListIndexFields() (tables []string) { + for indexName, index := range ps.i2f { + for fieldname, _ := range index.Fields { + + tables = append(tables, indexName+"/"+fieldname) + } + } + sort.Strings(tables) + return tables +} diff --git a/cmd/sauron/populate.sh b/cmd/sauron/populate.sh new file mode 100644 index 000000000..46e7dae67 --- /dev/null +++ b/cmd/sauron/populate.sh @@ -0,0 +1,10 @@ +#!/bin/bash +mod=20 +for i in `seq 100 1`;do +b=$(dc -e "$i $mod %p") +[[ $b == 0 ]] && mod=$(($mod+1)) +r=$(($b+30)) +payload="Set($i, j=$r)" +echo $payload +curl -XPOST "localhost:10101/index/i/query" -d "$payload" +done diff --git a/cmd/sauron/sample_schema.json b/cmd/sauron/sample_schema.json new file mode 100644 index 000000000..4dcc2c501 --- /dev/null +++ b/cmd/sauron/sample_schema.json @@ -0,0 +1 @@ +{"indexes":[{"name":"trait_store","createdAt":1595896639330112903,"options":{"keys":true,"trackExistence":true},"fields":[{"name":"aba","createdAt":1595896641512346986,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"ach_batch_recency","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_pass_thru_recency","createdAt":1595896640333428009,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"ach_payment_recency","createdAt":1595896642230184052,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"bools","createdAt":1595896640151751600,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"bools-exists","createdAt":1595896640855732760,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"central_group","createdAt":1595896640036032355,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"custom_audiences","createdAt":1595896643284443107,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"days_since_last_logon","createdAt":1595896639332677618,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":1,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"db","createdAt":1595896639548206413,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"desktop_recency","createdAt":1595896642633713852,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"domestic_wire_recency","createdAt":1595896642264402807,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"external_transfer_recency","createdAt":1595896639332854139,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"fields","createdAt":1595896639552562813,"options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"funds_transfer_recency","createdAt":1595896640803192694,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"international_wire_recency","createdAt":1595896643511015744,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"mobile_remote_deposit_recency","createdAt":1595896641174868205,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"payroll_recency","createdAt":1595896642880416952,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"pfm_category_total_current_balance__401k_investment","createdAt":1595896641492935339,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__403b_investment","createdAt":1595896639553865549,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__529_investment","createdAt":1595896643478533291,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__any","createdAt":1595896642202343868,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":31,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__auto_loan","createdAt":1595896642260798767,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__brokerage_account","createdAt":1595896639550130078,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__certificate_of_deposit","createdAt":1595896643210723905,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__checking","createdAt":1595896640763548235,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":42,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__credit_card","createdAt":1595896639734800167,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__home_equity_loan","createdAt":1595896642644368353,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__ira_investment","createdAt":1595896642076795921,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__line_of_credit","createdAt":1595896642071432710,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":32,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__loan","createdAt":1595896643101755055,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":35,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__money_market","createdAt":1595896642574933677,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":36,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__mortgage","createdAt":1595896640934767155,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__personal_loan","createdAt":1595896641378825138,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":29,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__roth_ira_investment","createdAt":1595896642821349899,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__savings","createdAt":1595896639534597334,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__simple_ira","createdAt":1595896639952200646,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":33,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__student_loan","createdAt":1595896642329394180,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"pfm_category_total_current_balance__taxable_investment","createdAt":1595896641406468398,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":30,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"phone_recency","createdAt":1595896639708002303,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_count__brokerage_account","createdAt":1595896642501159753,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_cd_or_share_certificate","createdAt":1595896643269910233,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_checking_or_share_draft","createdAt":1595896640044097611,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_credit_card","createdAt":1595896641500609762,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_indirect_auto_loan","createdAt":1595896643353298900,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_line_of_credit","createdAt":1595896641219946523,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_loan","createdAt":1595896641263557904,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_money_market","createdAt":1595896642506172994,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_new_auto_loan","createdAt":1595896640791087766,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_real_estate_loan","createdAt":1595896642616627035,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_savings_or_share","createdAt":1595896642313967229,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_credit_card","createdAt":1595896642254165546,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_secured_loan","createdAt":1595896642870853670,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_checking_or_share_draft","createdAt":1595896639333642847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_credit_card","createdAt":1595896642761030299,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_savings_or_share","createdAt":1595896643227769780,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_special_vehicle_loan","createdAt":1595896642991171391,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__commercial_used_auto_loan","createdAt":1595896642865599530,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_basic_checking_or_share_draft","createdAt":1595896640110059946,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":20,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_cd_or_share_certificate","createdAt":1595896640174468431,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_credit_card","createdAt":1595896640192571502,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_indirect_auto_loan","createdAt":1595896640563441892,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896643543919820,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_line_of_credit","createdAt":1595896640188449923,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_loan","createdAt":1595896639761090179,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_money_market","createdAt":1595896642731261748,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_new_auto_loan","createdAt":1595896643143947051,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_checking_or_share_draft","createdAt":1595896640346982139,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_premium_savings_or_share","createdAt":1595896641346726397,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_real_estate_loan","createdAt":1595896643351578694,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_checking_or_share_draft","createdAt":1595896640302899735,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_restricted_savings_or_share","createdAt":1595896642726551847,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_savings_or_share","createdAt":1595896642961872488,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_credit_card","createdAt":1595896640022927417,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_secured_loan","createdAt":1595896642352438551,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_checking_or_share_draft","createdAt":1595896640804808345,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_credit_card","createdAt":1595896641113696262,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_savings_or_share","createdAt":1595896640312498993,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_special_vehicle_loan","createdAt":1595896639886308884,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__consumer_used_auto_loan","createdAt":1595896642412302198,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__debit_card","createdAt":1595896643127876947,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__education_savings_or_share","createdAt":1595896639338093281,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__escrow_account","createdAt":1595896642969874721,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__health_savings_account","createdAt":1595896640570458731,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_line_of_credit","createdAt":1595896642592520668,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__home_equity_loan","createdAt":1595896643091627581,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__insurance_product","createdAt":1595896639744476133,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__interest_only_legal_trust_account","createdAt":1595896643005843869,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__membership_share","createdAt":1595896641732585653,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__other_service","createdAt":1595896641807551269,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_401k","createdAt":1595896639333271302,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":11,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_403b","createdAt":1595896643165638942,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__plan_529","createdAt":1595896641640608805,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira","createdAt":1595896639555618615,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":14,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__roth_ira_cd_or_share_certificate","createdAt":1595896641504460031,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":17,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__small_business_checking_or_share_draft","createdAt":1595896639332189957,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":19,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__special_indirect_vehicle_loan","createdAt":1595896641880199490,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":13,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__student_education_loan","createdAt":1595896639551558423,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira","createdAt":1595896643372890708,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":15,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__traditional_ira_cd_or_share_certificate","createdAt":1595896641621211665,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__trust","createdAt":1595896640709548779,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":18,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_count__vacation_club_holiday_savings","createdAt":1595896642256997553,"options":{"type":"decimal","base":0,"scale":3,"bitDepth":16,"min":-9223372036854775.808,"max":9223372036854775.807,"keys":false}},{"name":"product_recency__brokerage_account","createdAt":1595896641955724697,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_cd_or_share_certificate","createdAt":1595896642377298176,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_checking_or_share_draft","createdAt":1595896641640736297,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_credit_card","createdAt":1595896641199968081,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_indirect_auto_loan","createdAt":1595896643314414987,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_line_of_credit","createdAt":1595896641576267257,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_loan","createdAt":1595896641355763147,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_money_market","createdAt":1595896639970740640,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_new_auto_loan","createdAt":1595896642844117593,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_real_estate_loan","createdAt":1595896642953871689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_savings_or_share","createdAt":1595896643325231815,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_credit_card","createdAt":1595896642698454456,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_secured_loan","createdAt":1595896641951407002,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_checking_or_share_draft","createdAt":1595896642711316173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_credit_card","createdAt":1595896642147545663,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_savings_or_share","createdAt":1595896642364919307,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_special_vehicle_loan","createdAt":1595896639876538030,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__commercial_used_auto_loan","createdAt":1595896643527381053,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_basic_checking_or_share_draft","createdAt":1595896639874545938,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_cd_or_share_certificate","createdAt":1595896643147292944,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_credit_card","createdAt":1595896642605033674,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_indirect_auto_loan","createdAt":1595896641097157897,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_interest_bearing_checking_or_share_draft","createdAt":1595896642523867033,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_line_of_credit","createdAt":1595896641979726767,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_loan","createdAt":1595896639741458231,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_money_market","createdAt":1595896642350195677,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_new_auto_loan","createdAt":1595896642852653224,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_checking_or_share_draft","createdAt":1595896641391887646,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_premium_savings_or_share","createdAt":1595896643501816261,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_real_estate_loan","createdAt":1595896643015250459,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_checking_or_share_draft","createdAt":1595896640413965217,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_restricted_savings_or_share","createdAt":1595896642440765022,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_savings_or_share","createdAt":1595896643523788186,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_credit_card","createdAt":1595896641228514181,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_secured_loan","createdAt":1595896642815200189,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_checking_or_share_draft","createdAt":1595896639648900382,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_credit_card","createdAt":1595896639596071703,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_savings_or_share","createdAt":1595896642447703684,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_special_vehicle_loan","createdAt":1595896639863224318,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__consumer_used_auto_loan","createdAt":1595896639337315659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__debit_card","createdAt":1595896641091084110,"options":{"type":"int","base":0,"bitDepth":1,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__education_savings_or_share","createdAt":1595896643243122929,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__escrow_account","createdAt":1595896642288649376,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__health_savings_account","createdAt":1595896639713687734,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_line_of_credit","createdAt":1595896640644328270,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__home_equity_loan","createdAt":1595896643005117660,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__insurance_product","createdAt":1595896640080338115,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__interest_only_legal_trust_account","createdAt":1595896640710544689,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__membership_share","createdAt":1595896640920327282,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__other_service","createdAt":1595896641868683380,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_401k","createdAt":1595896643214447056,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_403b","createdAt":1595896639331510173,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__plan_529","createdAt":1595896641093053659,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira","createdAt":1595896640406186395,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__roth_ira_cd_or_share_certificate","createdAt":1595896642473602012,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__small_business_checking_or_share_draft","createdAt":1595896640631139451,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__special_indirect_vehicle_loan","createdAt":1595896641787887178,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__student_education_loan","createdAt":1595896640496522510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira","createdAt":1595896643383850510,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__traditional_ira_cd_or_share_certificate","createdAt":1595896642145329304,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__trust","createdAt":1595896640506789797,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"product_recency__vacation_club_holiday_savings","createdAt":1595896642206110046,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"stop_payment_recency","createdAt":1595896640954094741,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"survey_2053c210_adfc_4e14_afd3_9abc68a70719","createdAt":1595896642841258189,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2679bd7e_cdb8_4c24_a1da_e904799382eb","createdAt":1595896642498273724,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_2f336120_7130_492b_b116_d93d560baa0a","createdAt":1595896639333711539,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_36e229ff_c0a6_48ad_928e_19b5abf69f11","createdAt":1595896642987004213,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa","createdAt":1595896643012024691,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796","createdAt":1595896640184709609,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_587ae933_b8e3_49a2_a23d_989df9ad119f","createdAt":1595896642862965911,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab","createdAt":1595896642292355510,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949","createdAt":1595896640788795677,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_9cebb459_162e_4518_88f6_bc1cdd630cd0","createdAt":1595896642343098690,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_a05de6d3_6620_4eb2_b9a2_511936680b57","createdAt":1595896643389135517,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_b570363c_ba94_4a7a_9891_141a1befc42b","createdAt":1595896643120229702,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_c358d6e0_4116_4230_84ed_869b315e89a1","createdAt":1595896642202109079,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_cce773ce_14e8_4fae_90cd_213a821a3bc0","createdAt":1595896640040535589,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_e49df858_4903_40ac_bd9f_96f764940f2c","createdAt":1595896642346035396,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9","createdAt":1595896640643626147,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}},{"name":"tablet_recency","createdAt":1595896640414042095,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"user_id","createdAt":1595896640846590883,"options":{"type":"int","base":0,"bitDepth":21,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}},{"name":"zip_code","createdAt":1595896640534832822,"options":{"type":"mutex","cacheType":"ranked","cacheSize":50000,"keys":true}}],"shardWidth":1048576}]} diff --git a/cmd/sauron/sauron_test.go b/cmd/sauron/sauron_test.go new file mode 100644 index 000000000..53cf786de --- /dev/null +++ b/cmd/sauron/sauron_test.go @@ -0,0 +1,396 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strconv" + "testing" + + pb "github.com/golang/protobuf/proto" //nolint:staticcheck + pbuf "github.com/molecula/go-pilosa/v2/gopilosa_pbuf" + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/encoding/proto" + "github.com/pkg/errors" +) + +func TestSauron(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //remove the existing db file if present + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + + schemaJson := `{"indexes":[{"name":"scratch","createdAt":1611185870149882000,"options":{"keys":false,"trackExistence":true}, "fields":[{"name":"luminosity","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}}]}]}` + + client := &http.Client{} + t.Run("Schema", func(t *testing.T) { + _, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) + panicOn(err) + // verify that we can fetch it back + resp, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp.Body) + panicOn(err) + sbody := string(body) + if sbody[:62] != schemaJson[:62] { + fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) + panic("unexpected schema back") + } + }) + // add a set field "color" + //POST localhost:10101/index/scratch/field/color + t.Run("Create Non-Keyed SetField", func(t *testing.T) { + resp, err := client.Post(url+"/index/scratch/field/rating", "application/text", bytes.NewBuffer(nil)) + panicOn(err) + if resp.StatusCode != 200 { + panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) + } + }) + t.Run("Create Keyed SetField", func(t *testing.T) { + resp, err := client.Post(url+"/index/scratch/field/color", "application/text", bytes.NewBuffer([]byte(`{"options": {"keys": true}}`))) + panicOn(err) + if resp.StatusCode != 200 { + panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) + } + }) + t.Run("Create Time Field", func(t *testing.T) { + resp, err := client.Post(url+"/index/scratch/field/event", "application/text", bytes.NewBuffer([]byte(`{"options": { "type": "time", "timeQuantum": "YMDH" }}`))) + panicOn(err) + if resp.StatusCode != 200 { + panic(fmt.Sprintf("expected 200 status, got '%v'", resp)) + } + }) + + // set some bits + + // set some bits + + t.Run("Set", func(t *testing.T) { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) + panicOn(err) + // test for dups + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`)) + panicOn(err) + // + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=red)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=red)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=green)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=green)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=green)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=green)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=yellow)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=yellow)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=yellow)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=yellow)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=orange)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=orange)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=orange)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=orange)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, luminosity=1)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, luminosity=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, luminosity=4)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, luminosity=5)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, luminosity=4)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, luminosity=3)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, luminosity=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, luminosity=1)`)) + panicOn(err) + + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, rating=1)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(10, rating=1)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, rating=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, rating=2)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, rating=3)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, rating=4)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, rating=4)`)) + panicOn(err) + + //time fields + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T01:00)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T02:00)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, event=1,2021-02-05T02:05)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, event=1,2021-02-05T03:00)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, event=1,2021-02-05T04:00)`)) + panicOn(err) + }) + t.Run("Clear", func(t *testing.T) { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Clear(1, color=purple)`)) + panicOn(err) + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`ClearRow(color=purple)`)) + panicOn(err) + }) + t.Run("Store", func(t *testing.T) { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Store(Row(color=red), color=purple)`)) + panicOn(err) + }) + // verify that we can fetch it back + var schema pilosa.Schema + t.Run("Fetch Schema", func(t *testing.T) { + resp, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp.Body) + panicOn(err) + err = json.Unmarshal(body, &schema) + if err != nil { + panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) + } + }) + t.Run("Import Roaring", func(t *testing.T) { + //sets 12 bits in shard 0 + roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + idx := schema.Indexes[0] + var fld *pilosa.FieldInfo + for _, f := range idx.Fields { + if f.Name == "rating" { + fld = f + } + } + if fld == nil { + panic("field color not found") + } + msg := pilosa.ImportRoaringRequest{ + IndexCreatedAt: idx.CreatedAt, + FieldCreatedAt: fld.CreatedAt, + Clear: false, + Views: map[string][]byte{ + "": roaringData, + }, + UpdateExistence: true, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + furl := url + "/index/scratch/field/rating/import-roaring/0" + req, err := http.NewRequest("POST", furl, bytes.NewBuffer(data)) + panicOn(err) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + + // maybe need to Accept: "application/x-protobuf" + resp, err := client.Do(req) + panicOn(err) + _ = resp + }) + t.Run("Read Query", func(t *testing.T) { + for _, tst := range GetTests() { + _, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(tst.Pql)) + panicOn(err) + } + }) +} + +func makeImportRequest(field *FieldInfo2, shard uint64, rows, columns []uint64, clear bool) (path string, data []byte, err error) { + msg := &pbuf.ImportRequest{ + Index: field.GetIndexName(), + IndexCreatedAt: field.GetIndexCreatedAt(), + Field: field.GetName(), + FieldCreatedAt: field.GetCreatedAt(), + Shard: shard, + RowIDs: rows, + ColumnIDs: columns, + } + data, err = pb.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling Import to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) + return path, data, nil +} +func makeImportKeyRequest(field *FieldInfo2, shard uint64, rows []string, columns []uint64, clear bool) (path string, data []byte, err error) { + msg := &pbuf.ImportRequest{ + Index: field.GetIndexName(), + IndexCreatedAt: field.GetIndexCreatedAt(), + Field: field.GetName(), + FieldCreatedAt: field.GetCreatedAt(), + Shard: shard, + RowKeys: rows, + ColumnIDs: columns, + } + data, err = pb.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling Import to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) + return path, data, nil +} + +func makeImportValueRequest(field *FieldInfo2, shard uint64, values []int64, columns []uint64, clear bool) (path string, data []byte, err error) { + msg := &pbuf.ImportValueRequest{ + Index: field.GetIndexName(), + IndexCreatedAt: field.GetIndexCreatedAt(), + Field: field.GetName(), + FieldCreatedAt: field.GetCreatedAt(), + Shard: shard, + Values: values, + ColumnIDs: columns, + } + data, err = pb.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling Import to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear)) + return path, data, nil +} +func TestSauronImport(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //remove the existing db file if present + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + schemaJson := `{ + "indexes": [{ + "name": "scratch", + "createdAt": 1611185870149882000, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [{ + "name": "bsi", + "createdAt": 1595896639332730413, + "options": { + "type": "int", + "base": 0, + "bitDepth": 31, + "min": -9223372036854775808, + "max": 9223372036854775807, + "keys": false, + "foreignIndex": "" + } + }, { + "name": "decimal", + "createdAt": 1613177295654228860, + "options": { + "type": "decimal", + "base": 0, + "scale": 1, + "bitDepth": 0, + "min": -922337203685477580.8, + "max": 922337203685477580.7, + "keys": false + } + },{ + "name": "setkeyfield", + "createdAt": 1613345793598357200, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + },{ + "name": "setfield", + "createdAt": 1613345793598357200, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }] + }] +}` + client := &http.Client{} + _, err = client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) + panicOn(err) + + t.Run("basic row/column", func(t *testing.T) { + f, err := proxy.sql.GetField("scratch", "setfield") + panicOn(err) + clear := false + cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} + rows := []uint64{0, 0, 0, 1, 1, 1, 2, 2, 2} + shard := uint64(0) + path, payload, err := makeImportRequest(f, shard, rows, cols, clear) + _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) + panicOn(err) + }) + t.Run("basic rowkey/column", func(t *testing.T) { + f, err := proxy.sql.GetField("scratch", "setkeyfield") + panicOn(err) + clear := false + cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3} + rows := []string{"red", "red", "red", "blue", "blue", "blue", "green", "green", "green"} + shard := uint64(0) + path, payload, err := makeImportKeyRequest(f, shard, rows, cols, clear) + _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) + panicOn(err) + }) + t.Run("basic import values", func(t *testing.T) { + f, err := proxy.sql.GetField("scratch", "bsi") + panicOn(err) + clear := false + cols := []uint64{0, 1, 2, 3, 4} + values := []int64{40, 30, 20, 10, 5} + shard := uint64(0) + path, payload, err := makeImportValueRequest(f, shard, values, cols, clear) + _, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload)) + panicOn(err) + }) + +} diff --git a/cmd/sauron/schema_test.go b/cmd/sauron/schema_test.go new file mode 100644 index 000000000..23f5e924b --- /dev/null +++ b/cmd/sauron/schema_test.go @@ -0,0 +1,353 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/pilosa/pilosa/v2" +) + +func NewSauronTestConfig(t *testing.T) *SauronConfig { + + bind := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) + bindGRPC := fmt.Sprintf("127.0.0.1:%d", GetAvailPort()) + gossipPort := fmt.Sprintf("%d", GetAvailPort()) + dataDir, err := ioutil.TempDir("", "pilosa-sauron-target-*") + panicOn(err) + + return &SauronConfig{ + DatabaseName: strings.ToLower(t.Name()), + Bind: bind, + BindGRPC: bindGRPC, + GossipPort: gossipPort, + DataDir: dataDir, + } +} + +func TestProxyCanHandlePostSchema(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //start with + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + + // prep for test by cleaning up leftovers from any old run. + //panicOn(proxy.sql.DropTablesWithPrefix(t.Name())) + + //schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } }, { "name ": "color", "options ": {} } ] }] }` + /* + schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } } ] }] }` + + + vv("Posting err = '%v'", schemaJson) + resp, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson)) + if resp.StatusCode != http.StatusNoContent { + panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) + } + // verify that we can fetch it back + resp2, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp2.Body) + panicOn(err) + sbody := string(body) + vv("body = '%v'", sbody) + if sbody[:62] != schemaJson[:62] { + fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson) + panic("unxpected schema back") + } + */ + // load large schema too + client := &http.Client{} + schm, err := ioutil.ReadFile("sample_schema.json") + panicOn(err) + + resp, err := client.Post(url+"/schema", "application/json", bytes.NewBuffer(schm)) + panicOn(err) + if resp.StatusCode != http.StatusNoContent { + panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode)) + } + + // verify all tables expected are present + obs := make(map[string]bool) + for i, table := range proxy.sql.ListIndexFields() { + _ = i + obs[table] = true + if !expectedTables[table] { + panic(fmt.Sprintf("observed table '%v' but was not expected", table)) + } + } + for table := range expectedTables { + if !obs[table] { + panic(fmt.Sprintf("expected table '%v' but was not observed", table)) + } + } + +} + +func TestProxyCanCreateIndexFieldViaPost(t *testing.T) { + + cfg := NewSauronTestConfig(t) + proxy := NewSauron(cfg) + proxy.sql.MustRemove() //start with + url, err := proxy.Start() + panicOn(err) + defer proxy.Stop() + client := &http.Client{} + //create non-keyed or "simple" index + resp, err := client.Post(url+"/index/simple", "application/json", bytes.NewBuffer(nil)) + panicOn(err) + if resp.StatusCode != http.StatusOK { + panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) + } + + fields := []struct { + name string + options []byte + }{ + {name: "set", options: []byte{}}, + {name: "keyset", options: []byte{}}, + {name: "bsi", options: []byte{}}, + } + for _, field := range fields { + //create set field + resp, err = client.Post(fmt.Sprintf("%v/index/simple/field/%v", url, field.name), "application/json", bytes.NewBuffer(field.options)) + panicOn(err) + if resp.StatusCode != http.StatusOK { + panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode)) + } + } + // verify that we can fetch it back + resp2, err := client.Get(url + "/schema") + panicOn(err) + body, err := ioutil.ReadAll(resp2.Body) + panicOn(err) + var schema pilosa.Schema + err = json.Unmarshal(body, &schema) + if err != nil { + panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err)) + } + + expected := map[string]map[string]bool{ + "simple": {"bsi": true, "set": true, "keyset": true}, + } + for _, i := range schema.Indexes { + ll, ok := expected[i.Name] + if !ok { + panic(fmt.Sprintf("expected index not present: %v", i.Name)) + } + for _, f := range i.Fields { + _, ok := ll[f.Name] + if !ok { + panic(fmt.Sprintf("expected field '%v' not present in index'%v'", f.Name, i.Name)) + } + } + } +} + +var expectedTables = map[string]bool{ + "trait_store/aba": true, + "trait_store/ach_batch_recency": true, + "trait_store/ach_pass_thru_recency": true, + "trait_store/ach_payment_recency": true, + "trait_store/bools": true, + "trait_store/bools-exists": true, + "trait_store/central_group": true, + "trait_store/custom_audiences": true, + "trait_store/days_since_last_logon": true, + "trait_store/db": true, + "trait_store/desktop_recency": true, + "trait_store/domestic_wire_recency": true, + "trait_store/external_transfer_recency": true, + "trait_store/fields": true, + "trait_store/funds_transfer_recency": true, + "trait_store/international_wire_recency": true, + "trait_store/mobile_remote_deposit_recency": true, + "trait_store/payroll_recency": true, + "trait_store/pfm_category_total_current_balance__401k_investment": true, + "trait_store/pfm_category_total_current_balance__403b_investment": true, + "trait_store/pfm_category_total_current_balance__529_investment": true, + "trait_store/pfm_category_total_current_balance__any": true, + "trait_store/pfm_category_total_current_balance__auto_loan": true, + "trait_store/pfm_category_total_current_balance__brokerage_account": true, + "trait_store/pfm_category_total_current_balance__certificate_of_deposit": true, + "trait_store/pfm_category_total_current_balance__checking": true, + "trait_store/pfm_category_total_current_balance__credit_card": true, + "trait_store/pfm_category_total_current_balance__home_equity_loan": true, + "trait_store/pfm_category_total_current_balance__ira_investment": true, + "trait_store/pfm_category_total_current_balance__line_of_credit": true, + "trait_store/pfm_category_total_current_balance__loan": true, + "trait_store/pfm_category_total_current_balance__money_market": true, + "trait_store/pfm_category_total_current_balance__mortgage": true, + "trait_store/pfm_category_total_current_balance__personal_loan": true, + "trait_store/pfm_category_total_current_balance__roth_ira_investment": true, + "trait_store/pfm_category_total_current_balance__savings": true, + "trait_store/pfm_category_total_current_balance__simple_ira": true, + "trait_store/pfm_category_total_current_balance__student_loan": true, + "trait_store/pfm_category_total_current_balance__taxable_investment": true, + "trait_store/phone_recency": true, + "trait_store/product_count__brokerage_account": true, + "trait_store/product_count__commercial_cd_or_share_certificate": true, + "trait_store/product_count__commercial_checking_or_share_draft": true, + "trait_store/product_count__commercial_credit_card": true, + "trait_store/product_count__commercial_indirect_auto_loan": true, + "trait_store/product_count__commercial_line_of_credit": true, + "trait_store/product_count__commercial_loan": true, + "trait_store/product_count__commercial_money_market": true, + "trait_store/product_count__commercial_new_auto_loan": true, + "trait_store/product_count__commercial_real_estate_loan": true, + "trait_store/product_count__commercial_savings_or_share": true, + "trait_store/product_count__commercial_secured_credit_card": true, + "trait_store/product_count__commercial_secured_loan": true, + "trait_store/product_count__commercial_special_checking_or_share_draft": true, + "trait_store/product_count__commercial_special_credit_card": true, + "trait_store/product_count__commercial_special_savings_or_share": true, + "trait_store/product_count__commercial_special_vehicle_loan": true, + "trait_store/product_count__commercial_used_auto_loan": true, + "trait_store/product_count__consumer_basic_checking_or_share_draft": true, + "trait_store/product_count__consumer_cd_or_share_certificate": true, + "trait_store/product_count__consumer_credit_card": true, + "trait_store/product_count__consumer_indirect_auto_loan": true, + "trait_store/product_count__consumer_interest_bearing_checking_or_share_draft": true, + "trait_store/product_count__consumer_line_of_credit": true, + "trait_store/product_count__consumer_loan": true, + "trait_store/product_count__consumer_money_market": true, + "trait_store/product_count__consumer_new_auto_loan": true, + "trait_store/product_count__consumer_premium_checking_or_share_draft": true, + "trait_store/product_count__consumer_premium_savings_or_share": true, + "trait_store/product_count__consumer_real_estate_loan": true, + "trait_store/product_count__consumer_restricted_checking_or_share_draft": true, + "trait_store/product_count__consumer_restricted_savings_or_share": true, + "trait_store/product_count__consumer_savings_or_share": true, + "trait_store/product_count__consumer_secured_credit_card": true, + "trait_store/product_count__consumer_secured_loan": true, + "trait_store/product_count__consumer_special_checking_or_share_draft": true, + "trait_store/product_count__consumer_special_credit_card": true, + "trait_store/product_count__consumer_special_savings_or_share": true, + "trait_store/product_count__consumer_special_vehicle_loan": true, + "trait_store/product_count__consumer_used_auto_loan": true, + "trait_store/product_count__debit_card": true, + "trait_store/product_count__education_savings_or_share": true, + "trait_store/product_count__escrow_account": true, + "trait_store/product_count__health_savings_account": true, + "trait_store/product_count__home_equity_line_of_credit": true, + "trait_store/product_count__home_equity_loan": true, + "trait_store/product_count__insurance_product": true, + "trait_store/product_count__interest_only_legal_trust_account": true, + "trait_store/product_count__membership_share": true, + "trait_store/product_count__other_service": true, + "trait_store/product_count__plan_401k": true, + "trait_store/product_count__plan_403b": true, + "trait_store/product_count__plan_529": true, + "trait_store/product_count__roth_ira": true, + "trait_store/product_count__roth_ira_cd_or_share_certificate": true, + "trait_store/product_count__small_business_checking_or_share_draft": true, + "trait_store/product_count__special_indirect_vehicle_loan": true, + "trait_store/product_count__student_education_loan": true, + "trait_store/product_count__traditional_ira": true, + "trait_store/product_count__traditional_ira_cd_or_share_certificate": true, + "trait_store/product_count__trust": true, + "trait_store/product_count__vacation_club_holiday_savings": true, + "trait_store/product_recency__brokerage_account": true, + "trait_store/product_recency__commercial_cd_or_share_certificate": true, + "trait_store/product_recency__commercial_checking_or_share_draft": true, + "trait_store/product_recency__commercial_credit_card": true, + "trait_store/product_recency__commercial_indirect_auto_loan": true, + "trait_store/product_recency__commercial_line_of_credit": true, + "trait_store/product_recency__commercial_loan": true, + "trait_store/product_recency__commercial_money_market": true, + "trait_store/product_recency__commercial_new_auto_loan": true, + "trait_store/product_recency__commercial_real_estate_loan": true, + "trait_store/product_recency__commercial_savings_or_share": true, + "trait_store/product_recency__commercial_secured_credit_card": true, + "trait_store/product_recency__commercial_secured_loan": true, + "trait_store/product_recency__commercial_special_checking_or_share_draft": true, + "trait_store/product_recency__commercial_special_credit_card": true, + "trait_store/product_recency__commercial_special_savings_or_share": true, + "trait_store/product_recency__commercial_special_vehicle_loan": true, + "trait_store/product_recency__commercial_used_auto_loan": true, + "trait_store/product_recency__consumer_basic_checking_or_share_draft": true, + "trait_store/product_recency__consumer_cd_or_share_certificate": true, + "trait_store/product_recency__consumer_credit_card": true, + "trait_store/product_recency__consumer_indirect_auto_loan": true, + "trait_store/product_recency__consumer_interest_bearing_checking_or_share_draft": true, + "trait_store/product_recency__consumer_line_of_credit": true, + "trait_store/product_recency__consumer_loan": true, + "trait_store/product_recency__consumer_money_market": true, + "trait_store/product_recency__consumer_new_auto_loan": true, + "trait_store/product_recency__consumer_premium_checking_or_share_draft": true, + "trait_store/product_recency__consumer_premium_savings_or_share": true, + "trait_store/product_recency__consumer_real_estate_loan": true, + "trait_store/product_recency__consumer_restricted_checking_or_share_draft": true, + "trait_store/product_recency__consumer_restricted_savings_or_share": true, + "trait_store/product_recency__consumer_savings_or_share": true, + "trait_store/product_recency__consumer_secured_credit_card": true, + "trait_store/product_recency__consumer_secured_loan": true, + "trait_store/product_recency__consumer_special_checking_or_share_draft": true, + "trait_store/product_recency__consumer_special_credit_card": true, + "trait_store/product_recency__consumer_special_savings_or_share": true, + "trait_store/product_recency__consumer_special_vehicle_loan": true, + "trait_store/product_recency__consumer_used_auto_loan": true, + "trait_store/product_recency__debit_card": true, + "trait_store/product_recency__education_savings_or_share": true, + "trait_store/product_recency__escrow_account": true, + "trait_store/product_recency__health_savings_account": true, + "trait_store/product_recency__home_equity_line_of_credit": true, + "trait_store/product_recency__home_equity_loan": true, + "trait_store/product_recency__insurance_product": true, + "trait_store/product_recency__interest_only_legal_trust_account": true, + "trait_store/product_recency__membership_share": true, + "trait_store/product_recency__other_service": true, + "trait_store/product_recency__plan_401k": true, + "trait_store/product_recency__plan_403b": true, + "trait_store/product_recency__plan_529": true, + "trait_store/product_recency__roth_ira": true, + "trait_store/product_recency__roth_ira_cd_or_share_certificate": true, + "trait_store/product_recency__small_business_checking_or_share_draft": true, + "trait_store/product_recency__special_indirect_vehicle_loan": true, + "trait_store/product_recency__student_education_loan": true, + "trait_store/product_recency__traditional_ira": true, + "trait_store/product_recency__traditional_ira_cd_or_share_certificate": true, + "trait_store/product_recency__trust": true, + "trait_store/product_recency__vacation_club_holiday_savings": true, + "trait_store/stop_payment_recency": true, + "trait_store/survey_2053c210_adfc_4e14_afd3_9abc68a70719": true, + "trait_store/survey_2679bd7e_cdb8_4c24_a1da_e904799382eb": true, + "trait_store/survey_2f336120_7130_492b_b116_d93d560baa0a": true, + "trait_store/survey_36e229ff_c0a6_48ad_928e_19b5abf69f11": true, + "trait_store/survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa": true, + "trait_store/survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796": true, + "trait_store/survey_587ae933_b8e3_49a2_a23d_989df9ad119f": true, + "trait_store/survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab": true, + "trait_store/survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949": true, + "trait_store/survey_9cebb459_162e_4518_88f6_bc1cdd630cd0": true, + "trait_store/survey_a05de6d3_6620_4eb2_b9a2_511936680b57": true, + "trait_store/survey_b570363c_ba94_4a7a_9891_141a1befc42b": true, + "trait_store/survey_c358d6e0_4116_4230_84ed_869b315e89a1": true, + "trait_store/survey_cce773ce_14e8_4fae_90cd_213a821a3bc0": true, + "trait_store/survey_e49df858_4903_40ac_bd9f_96f764940f2c": true, + "trait_store/survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9": true, + "trait_store/tablet_recency": true, + "trait_store/user_id": true, + "trait_store/zip_code": true, +} diff --git a/cmd/sauron/sqlgen_test.go b/cmd/sauron/sqlgen_test.go new file mode 100644 index 000000000..b9d11a728 --- /dev/null +++ b/cmd/sauron/sqlgen_test.go @@ -0,0 +1,260 @@ +package main + +import ( + "strings" + "testing" + + "github.com/pilosa/pilosa/v2/pql" + "github.com/shurcooL/go-goon" +) + +/* +sqlite db this test validated this against +create table bits (field ,row , column , timestamp); + create table λbsi (field string,column bigint , val bigint); + CREATE VIEW columns as + select distinct column from λbits + UNION + select distinct column from λbsi; + + INSERT INTO λbits VALUES( 'color','red',1); + INSERT INTO λbits VALUES( 'color','red',2); + INSERT INTO λbits VALUES( 'color','red',3); + INSERT INTO λbits VALUES( 'color','red',4); + INSERT INTO λbits VALUES( 'color','red',5); + INSERT INTO λbits VALUES( 'color','green',2); + INSERT INTO λbits VALUES( 'color','green',4); + INSERT INTO λbits VALUES( 'color','green',6); + INSERT INTO λbits VALUES( 'color','green',8); + INSERT INTO λbits VALUES( 'color','yellow',1); + INSERT INTO λbits VALUES( 'color','yellow',3); + INSERT INTO λbits VALUES( 'color','yellow',5); + INSERT INTO λbits VALUES( 'color','yellow',7); + INSERT INTO λbits VALUES( 'color','orange',5); + INSERT INTO λbits VALUES( 'color','orange',6); + INSERT INTO λbits VALUES( 'color','orange',7); + INSERT INTO λbits VALUES( 'color','orange',8); + + INSERT INTO λbits VALUES( 'rating',1,8); + INSERT INTO λbits VALUES( 'rating',1,7); + INSERT INTO λbits VALUES( 'rating',1,10); + INSERT INTO λbits VALUES( 'rating',2,6); + INSERT INTO λbits VALUES( 'rating',2,5); + INSERT INTO λbits VALUES( 'rating',3,4); + INSERT INTO λbits VALUES( 'rating',4,3); + INSERT INTO λbits VALUES( 'rating',4,2); + + INSERT INTO λbsi VALUES( 'luminosity',1, 1); + INSERT INTO λbsi VALUES( 'luminosity',2, 2); + INSERT INTO λbsi VALUES( 'luminosity',3, 4); + INSERT INTO λbsi VALUES( 'luminosity',4, 5); + INSERT INTO λbsi VALUES( 'luminosity',5, 4); + INSERT INTO λbsi VALUES( 'luminosity',6, 3); + INSERT INTO λbsi VALUES( 'luminosity',7, 2); + INSERT INTO λbsi VALUES( 'luminosity',8, 1); +*/ +func GetTests() []struct { + Pql string + Sql string +} { + return []struct { + Pql string + Sql string + }{ + { + Pql: "Row(rating=0)", + Sql: `select distinct column from λbits where field="rating" AND row="0"`, + }, + { + Pql: "Row(color=red)", + Sql: `select distinct column from λbits where field="color" AND row="red"`, + }, + { + Pql: "Row(luminosity>2)", + Sql: `select distinct column from λbsi where field="luminosity" and val>2`, + }, + { + Pql: "Row(luminosity<2)", + Sql: `select distinct column from λbsi where field="luminosity" and val<2`, + }, + { + Pql: "Row(luminosity==4)", + Sql: `select distinct column from λbsi where field="luminosity" and val=4`, + }, + { + Pql: "Row(luminosity<=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val<=4`, + }, + { + Pql: "Row(luminosity>=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val>=4`, + }, + { + Pql: "Row(luminosity!=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val!=4`, + }, + { + Pql: "Row(2<=luminosity<=4)", + Sql: `select distinct column from λbsi where field="luminosity" and val>=2 and val<=4`, + }, + { + Pql: "Row(22 and val<=4`, + }, + { + Pql: "Row(22 and val<4`, + }, + { + Pql: `Row(event=1, from='2021-02-01T00:00', to='2021-02-05T03:00')`, + Sql: `select distinct column from λbits where field="event" AND row="1" AND timestamp >= "2021-02-01 00:00" AND timestamp < "2021-02-05 03:00"`, + }, + { + Pql: "Count(row(color=red))", + Sql: `select count(*) from( select distinct column from λbits where field="color" AND row="red" )`, + }, + { + Pql: `Intersect(Row(color=red),Row(color=green))`, + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: "Count(Union(Row(color=red),Row(color=green)))", + Sql: `select count(*) from( select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") )`, + }, + { + Pql: "Union(Row(color=red),Row(color=green))", + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: "Difference(Row(color=red),Row(color=green))", + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" except select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: `All()`, + Sql: `select column from λcolumns`, + }, + { + Pql: "Not(Row(color=red))", + Sql: `select column from λcolumns except select distinct column from λbits where field="color" AND row="red"`, + }, + { + Pql: "Xor(Row(color=red),Row(color=green))", + Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") except select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`, + }, + { + Pql: "Distinct(field=luminosity)", + Sql: `select distinct val from λbsi where field="luminosity"`, + }, + { + Pql: "Distinct(Row(color=red),field=luminosity)", + Sql: `select distinct val from λbsi where field="luminosity" AND column in (select distinct column from λbits where field="color" AND row="red")`, + }, + { + Pql: "Distinct(Intersect(Row(color=red),Row(color=green)),field=luminosity)", + Sql: `select distinct val from λbsi where field="luminosity" AND column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green"))`, + }, + { + Pql: "Rows(color)", + Sql: `select distinct field, row from λbits where field="color" order by row`, + }, + { + Pql: "Rows(color, limit=10)", + Sql: `select distinct field, row from λbits where field="color" order by row limit 10`, + }, + { + Pql: "Rows(color, column=1)", + Sql: `select distinct field, row from λbits where field="color" and column='1' order by row`, + }, + // { // TODO (twg) Not supported by this test, can't reliably get key ordering + // Pql: "Rows(color, previous=green )", + // Sql: `select distinct field, row from λbits where field="color" and row>'green' order by row`, + // }, + { + Pql: "GroupBy(Rows(color))", + Sql: `select f1.field, f1.row,count(*) as cnt from λbits f1 where f1.field='color' group by f1.field,f1.row order by cnt desc`, + }, + { + Pql: "GroupBy(Rows(color),Rows(rating))", + Sql: `select f1.field, f1.row,f2.field, f2.row,count(*) as cnt from λbits f1 inner join λbits f2 on f1.column = f2.column where f1.field='color'and f2.field='rating' group by f1.field,f1.row,f2.field,f2.row order by cnt desc`, + }, + { + Pql: "TopN(color)", + Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, + }, + { + Pql: "TopN(color,n=2)", + Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc limit 2`, + }, + { + Pql: "TopN(color,Row(color=red),n=2)", + Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="color" AND row="red") group by row order by cnt desc limit 2`, + }, + { + Pql: "Topk(color)", + Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`, + }, + { + Pql: "TopK(color, k=2, filter=Row(rating=1))", + Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="rating" AND row="1") group by row order by cnt desc limit 2`, + }, + { + Pql: `Min(field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity")`, + }, + { + Pql: `Max(field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select max(val) from λbsi where field="luminosity")`, + }, + { + Pql: `Min(Row(color=orange),field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select distinct column from λbits where field="color" AND row="orange")) and column in (select distinct column from λbits where field="color" AND row="orange")`, + }, + { + Pql: `Min(Intersect(Row(color=red),Row(color=orange)),field="luminosity")`, + Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))) and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))`, + }, + { + Pql: `Sum(field="luminosity")`, + Sql: `select sum(val),count(*) from λbsi where field="luminosity"`, + }, + { + Pql: `ConstRow(columns=[1,3,5])`, + Sql: `select column1 as column from (values (1),(3),(5))`, + }, + { + Pql: `UnionRows(Rows(color))`, + Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, + }, + { + Pql: `UnionRows(Rows(color),Rows(rating))`, + Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`, + }, + { + Pql: `IncludesColumn(Row(color=red), column=1)`, + Sql: `select count(*)>0 from (select column from(select distinct column from λbits where field="color" AND row="red") where column='1')`, + }, + { + Pql: `Extract(Row(color=red),Rows(color),Rows(rating))`, + Sql: `select sq.field, sq.row, b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row where b.column in (select distinct column from λbits where field="color" AND row="red")`, + }, + { + Pql: `Extract(ConstRow(columns=[1,2,3]))`, + Sql: `select '' as field, '' as row,column from (select column1 as column from (values (1),(2),(3)))`, + }, + } +} +func TestParse(t *testing.T) { + + for _, tst := range GetTests() { + + q, err := pql.NewParser(strings.NewReader(tst.Pql)).Parse() + panicOn(err) + sql, _, _, err := ToSql(q.Calls, "") + panicOn(err) + if sql != tst.Sql { + vv("\npql:%v\ngenerated:\n%v\nexpected:\n%v\n", tst.Pql, sql, tst.Sql) + goon.Dump(q) + } + + } +} diff --git a/cmd/sauron/test_sqlite.sh b/cmd/sauron/test_sqlite.sh new file mode 100755 index 000000000..d0ab85b2f --- /dev/null +++ b/cmd/sauron/test_sqlite.sh @@ -0,0 +1,51 @@ +#!/bin/bash +db="poc2.db" +sqlite3 ${db} < 0 { + frames := runtime.CallersFrames(pc[:n]) + for i := 0; i <= target; i++ { + contender, more := frames.Next() + if i == target { + f = contender + } + if !more { + break + } + } + } + return f.Function +}