From f4385df2cf2bc80a287cc9259d4eb39a98ea56da Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 21 Nov 2022 18:43:53 -0600 Subject: [PATCH] Fix formatting in CLI results with custom SQLResonse.UnmarshalJSON (#2305) * Fix formatting in CLI results with custom SQLResonse.UnmarshalJSON When I started this, it was meant to be a quick fix to address the confusing result formats we were seeing in the CLI. For example, all large integer values were displayed in scientifc notation. This is because we were passing the result types from JSON (in this case, float64) into pretty print. Similarly, `IDSets` and `StringSets` where being printed using the default go Stringer for the types []int64 and []string respectively. I started by writing a customer UnmarshalJSON() method for the `SQLResponse` type. Part of this (the part which converts data types based on header types) was already being used in dax tests, so this just formalizes that logic as part of the `SQLResponse` type. Then I realized that the sql3 tests (run against the `sql3` package) were failing because sql3 is not actually returning the `IDSets` and `StringSets` types. A future task is to formalize return types, define them, and modify sql3 to return them. Once that is done, we can remove the "typed" switch in the `SQLResponse` json unmarshaller. Another significant change is the modification to the `ExprDataType` interface: ``` type ExprDataType interface { exprDataType() TypeName() string TypeDescription() string TypeInfo() map[string]interface{} } ``` I added two more methods in order to distinguish between a type (`DECIMAL`), its description (`DECIMAL(2)`), and its type info (`"scale": int64(2)`). Currently, the description can be used as the field definition in a CREATE TABLE statement, but we may want to re-think that. Also, Decimal is the only type currently using TypeInfo. Finally, I tried to consilidate things around `dax.FieldType` instead of comparing against parser types outside of sql3. We still have some sql3 parser and planner types lurking about, but we can address those in future commits. * Add some test coverage * smoke test expected INT, now int * minor fixes * Introduce WireQueryResponse and related types This also changes dax.FieldType to dax.BaseType. * Populate WireQueryResponse correctly Currently this is in the http handler, and in the queryer. * Convert sql3 and dax tests to expect pilosa.WireQueryField in results * fix PQL tests in the SQL defs * Address a few of the skipped sql tests in dax --- api_directive.go | 16 +- ctl/cli.go | 30 +- dax/Makefile | 4 +- dax/mds/schemar/boltdb/schemar_test.go | 4 +- dax/mds/schemar/schemar_test.go | 4 +- dax/queryer/queryer.go | 57 ++-- dax/queryer/schema_api.go | 34 +-- dax/table.go | 59 ++-- dax/table_test.go | 6 +- dax/test/dax/dax_test.go | 146 ++-------- dax/test/table.go | 12 +- fbcloud/client.go | 4 +- http_handler.go | 29 +- idk/ingest_test.go | 18 +- idk/mds/schemamanager.go | 16 +- qa/testcases/smoketest/test_smoke.py | 5 +- sql.go | 9 - sql3/errors.go | 2 +- sql3/parser/astdatatype.go | 116 ++++++++ sql3/planner/expression.go | 10 +- sql3/planner/expressionanalyzer.go | 52 ++-- sql3/planner/expressionanalyzercall.go | 6 +- sql3/sql_complex_test.go | 368 ++++++++++++++----------- sql3/sql_test.go | 7 +- sql3/test/defs/defs_aggregate.go | 9 +- sql3/test/defs/defs_between.go | 20 +- sql3/test/defs/defs_groupby.go | 15 +- sql3/test/defs/types.go | 74 +++-- sql3/test/helpers.go | 24 +- sql_test.go | 95 +++++++ wire_response.go | 191 +++++++++++++ 31 files changed, 931 insertions(+), 511 deletions(-) delete mode 100644 sql.go create mode 100644 sql_test.go create mode 100644 wire_response.go diff --git a/api_directive.go b/api_directive.go index 2ecb3146a..296550c07 100644 --- a/api_directive.go +++ b/api_directive.go @@ -938,37 +938,37 @@ func createField(idx *Index, fld *dax.Field) error { opts := []FieldOption{} switch fld.Type { - case dax.FieldTypeBool: + case dax.BaseTypeBool: opts = append(opts, OptFieldTypeBool(), ) - case dax.FieldTypeDecimal: + case dax.BaseTypeDecimal: opts = append(opts, OptFieldTypeDecimal(fld.Options.Scale), ) - case dax.FieldTypeID: + case dax.BaseTypeID: opts = append(opts, OptFieldTypeMutex(cacheType, cacheSize), ) - case dax.FieldTypeIDSet: + case dax.BaseTypeIDSet: opts = append(opts, OptFieldTypeSet(cacheType, cacheSize), ) - case dax.FieldTypeInt: + case dax.BaseTypeInt: opts = append(opts, OptFieldTypeInt(fld.Options.Min.ToInt64(0), fld.Options.Max.ToInt64(0)), ) - case dax.FieldTypeString: + case dax.BaseTypeString: opts = append(opts, OptFieldTypeMutex(cacheType, cacheSize), OptFieldKeys(), ) - case dax.FieldTypeStringSet: + case dax.BaseTypeStringSet: opts = append(opts, OptFieldTypeSet(cacheType, cacheSize), OptFieldKeys(), ) - case dax.FieldTypeTimestamp: + case dax.BaseTypeTimestamp: opts = append(opts, OptFieldTypeTimestamp(fld.Options.Epoch, fld.Options.TimeUnit), ) diff --git a/ctl/cli.go b/ctl/cli.go index f66455ed2..82b212b24 100644 --- a/ctl/cli.go +++ b/ctl/cli.go @@ -319,7 +319,7 @@ func appendCommand(orig string, part string) string { } type FBQueryer interface { - Query(org, db, sql string) (*featurebase.SQLResponse, error) + Query(org, db, sql string) (*featurebase.WireQueryResponse, error) } func (cmd *CLICommand) executeCommands(ctx context.Context) error { @@ -341,7 +341,7 @@ func (cmd *CLICommand) executeCommands(ctx context.Context) error { fmt.Printf("making query: %v\n", err) continue } - err = WriteOut(sqlResponse, os.Stdout) + err = writeOut(sqlResponse, os.Stdout) if err != nil { return errors.Wrap(err, "writing out response") } @@ -401,7 +401,7 @@ func (cmd *CLICommand) handleIfNonSQLCommand(ctx context.Context, sql string) (b return handled, nil } -func WriteWarnings(r *featurebase.SQLResponse, w io.Writer) error { +func writeWarnings(r *featurebase.WireQueryResponse, w io.Writer) error { if len(r.Warnings) > 0 { if _, err := w.Write([]byte("\n")); err != nil { return errors.Wrapf(err, "writing warning: %s", r.Error) @@ -415,7 +415,7 @@ func WriteWarnings(r *featurebase.SQLResponse, w io.Writer) error { return nil } -func WriteOut(r *featurebase.SQLResponse, w io.Writer) error { +func writeOut(r *featurebase.WireQueryResponse, w io.Writer) error { if r == nil { return errors.New("attempt to write out nil response") } @@ -423,7 +423,7 @@ func WriteOut(r *featurebase.SQLResponse, w io.Writer) error { if _, err := w.Write([]byte("Error: " + r.Error + "\n")); err != nil { return errors.Wrapf(err, "writing error: %s", r.Error) } - return WriteWarnings(r, w) + return writeWarnings(r, w) } t := table.NewWriter() @@ -445,7 +445,7 @@ func WriteOut(r *featurebase.SQLResponse, w io.Writer) error { } t.Render() - err := WriteWarnings(r, w) + err := writeWarnings(r, w) if err != nil { return err } @@ -465,7 +465,7 @@ func WriteOut(r *featurebase.SQLResponse, w io.Writer) error { return nil } -func schemaToRow(schema featurebase.SQLSchema) []interface{} { +func schemaToRow(schema featurebase.WireQuerySchema) []interface{} { ret := make([]interface{}, len(schema.Fields)) for i, field := range schema.Fields { ret[i] = field.Name @@ -483,7 +483,7 @@ type standardQueryer struct { Port string } -func (qryr *standardQueryer) Query(org, db, sql string) (*featurebase.SQLResponse, error) { +func (qryr *standardQueryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) { buf := bytes.Buffer{} url := fmt.Sprintf("%s/sql", hostPort(qryr.Host, qryr.Port)) @@ -498,8 +498,10 @@ func (qryr *standardQueryer) Query(org, db, sql string) (*featurebase.SQLRespons if err != nil { return nil, errors.Wrap(err, "reading response") } - sqlResponse := &featurebase.SQLResponse{} - if err := json.Unmarshal(fullbod, sqlResponse); err != nil { + sqlResponse := &featurebase.WireQueryResponse{} + // TODO(tlt): switch this back once all responses are typed + // if err := json.Unmarshal(fullbod, sqlResponse); err != nil { + if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil { return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod) } @@ -517,7 +519,7 @@ type daxQueryer struct { Port string } -func (qryr *daxQueryer) Query(org, db, sql string) (*featurebase.SQLResponse, error) { +func (qryr *daxQueryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) { buf := bytes.Buffer{} url := fmt.Sprintf("%s/queryer/sql", hostPort(qryr.Host, qryr.Port)) @@ -539,8 +541,10 @@ func (qryr *daxQueryer) Query(org, db, sql string) (*featurebase.SQLResponse, er if err != nil { return nil, errors.Wrap(err, "reading response") } - sqlResponse := &featurebase.SQLResponse{} - if err := json.Unmarshal(fullbod, sqlResponse); err != nil { + sqlResponse := &featurebase.WireQueryResponse{} + // TODO(tlt): switch this back once all responses are typed + // if err := json.Unmarshal(fullbod, sqlResponse); err != nil { + if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil { return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod) } diff --git a/dax/Makefile b/dax/Makefile index 6eb0bc7f8..bef63ce73 100644 --- a/dax/Makefile +++ b/dax/Makefile @@ -15,10 +15,10 @@ testv: test-integration: mkdir -p ../coverage-from-docker - $(GO) test ./test/dax -count 1 -run Integration + $(GO) test ./test/dax -count 1 -run TestDAXIntegration testv-integration: - $(GO) test -v ./test/dax -count 1 -run Integration + $(GO) test -v ./test/dax -count 1 -run TestDAXIntegration diff --git a/dax/mds/schemar/boltdb/schemar_test.go b/dax/mds/schemar/boltdb/schemar_test.go index 4460e7d66..def03a0e6 100644 --- a/dax/mds/schemar/boltdb/schemar_test.go +++ b/dax/mds/schemar/boltdb/schemar_test.go @@ -46,11 +46,11 @@ func TestSchemar(t *testing.T) { tbl.Fields = []*dax.Field{ { Name: dax.PrimaryKeyFieldName, - Type: dax.FieldTypeString, + Type: dax.BaseTypeString, }, { Name: "intField", - Type: dax.FieldTypeInt, + Type: dax.BaseTypeInt, }, } qtbl := dax.NewQualifiedTable(qual, tbl) diff --git a/dax/mds/schemar/schemar_test.go b/dax/mds/schemar/schemar_test.go index 62a4f2edf..44f9a67fd 100644 --- a/dax/mds/schemar/schemar_test.go +++ b/dax/mds/schemar/schemar_test.go @@ -33,11 +33,11 @@ func TestSchemar(t *testing.T) { tbl.Fields = []*dax.Field{ { Name: dax.PrimaryKeyFieldName, - Type: dax.FieldTypeString, + Type: dax.BaseTypeString, }, { Name: "intField", - Type: dax.FieldTypeInt, + Type: dax.BaseTypeInt, }, } qtbl := dax.NewQualifiedTable(qual, tbl) diff --git a/dax/queryer/queryer.go b/dax/queryer/queryer.go index b73f4f3f5..bb66262a7 100644 --- a/dax/queryer/queryer.go +++ b/dax/queryer/queryer.go @@ -76,13 +76,13 @@ func New(cfg Config) *Queryer { return q } -func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql string) (*featurebase.SQLResponse, error) { +func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql string) (*featurebase.WireQueryResponse, error) { start := time.Now() if len(sql) > 0 && sql[0] == '[' { return q.parseAndQueryPQL(ctx, qual, sql) } - ret := &featurebase.SQLResponse{} + ret := &featurebase.WireQueryResponse{} applyExecutionTime := func() { ret.ExecutionTime = time.Since(start).Microseconds() @@ -133,13 +133,20 @@ func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql str // Read schema. columns := planOp.Schema() - schema := featurebase.SQLSchema{ - Fields: make([]*featurebase.SQLField, len(columns)), + schema := featurebase.WireQuerySchema{ + Fields: make([]*featurebase.WireQueryField, len(columns)), } for i, col := range columns { - schema.Fields[i] = &featurebase.SQLField{ - Name: col.ColumnName, - Type: col.Type.TypeName(), + btype, err := dax.BaseTypeFromString(col.Type.TypeName()) + if err != nil { + applyError(errors.Wrap(err, "getting fieldtype from string")) + return ret, nil + } + schema.Fields[i] = &featurebase.WireQueryField{ + Name: dax.FieldName(col.ColumnName), + Type: col.Type.TypeDescription(), + BaseType: btype, + TypeInfo: col.Type.TypeInfo(), } } @@ -161,7 +168,7 @@ func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql str return ret, nil } -func (q *Queryer) parseAndQueryPQL(ctx context.Context, qual dax.TableQualifier, sql string) (*featurebase.SQLResponse, error) { +func (q *Queryer) parseAndQueryPQL(ctx context.Context, qual dax.TableQualifier, sql string) (*featurebase.WireQueryResponse, error) { var i int for i = 1; sql[i] != ']'; i++ { if i == len(sql)-1 { @@ -175,7 +182,7 @@ func (q *Queryer) parseAndQueryPQL(ctx context.Context, qual dax.TableQualifier, return q.QueryPQL(ctx, qual, dax.TableName(table), query) } -func (q *Queryer) QueryPQL(ctx context.Context, qual dax.TableQualifier, table dax.TableName, pql string) (*featurebase.SQLResponse, error) { +func (q *Queryer) QueryPQL(ctx context.Context, qual dax.TableQualifier, table dax.TableName, pql string) (*featurebase.WireQueryResponse, error) { // Parse the pql into a pql.Query containing []pql.Call. qry, err := featurebase_pql.NewParser(strings.NewReader(pql)).Parse() if err != nil { @@ -201,7 +208,7 @@ func (q *Queryer) QueryPQL(ctx context.Context, qual dax.TableQualifier, table d return PQLResultToQueryResult(results.Results[0]) } -func PQLResultToQueryResult(pqlResult interface{}) (*featurebase.SQLResponse, error) { +func PQLResultToQueryResult(pqlResult interface{}) (*featurebase.WireQueryResponse, error) { toTabler, err := server.ToTablerWrapper(pqlResult) if err != nil { return nil, errors.Wrap(err, "wrapping as type ToTabler") @@ -214,13 +221,17 @@ func PQLResultToQueryResult(pqlResult interface{}) (*featurebase.SQLResponse, er return tableResponseToQueryResult(table) } -func tableResponseToQueryResult(t *fbproto.TableResponse) (*featurebase.SQLResponse, error) { - qr := &featurebase.SQLResponse{ - Schema: featurebase.SQLSchema{Fields: make([]*featurebase.SQLField, len(t.Headers))}, +func tableResponseToQueryResult(t *fbproto.TableResponse) (*featurebase.WireQueryResponse, error) { + qr := &featurebase.WireQueryResponse{ + Schema: featurebase.WireQuerySchema{Fields: make([]*featurebase.WireQueryField, len(t.Headers))}, Data: make([][]interface{}, len(t.Rows)), } for i, ci := range t.Headers { - qr.Schema.Fields[i] = &featurebase.SQLField{Name: ci.Name, Type: datatypeToType(ci.Datatype)} + qr.Schema.Fields[i] = &featurebase.WireQueryField{ + Name: dax.FieldName(ci.Name), + Type: string(datatypeToBaseType(ci.Datatype)), // TODO(tlt): this doesn't contain typeInfo + BaseType: datatypeToBaseType(ci.Datatype), + } } for i, row := range t.Rows { @@ -230,27 +241,27 @@ func tableResponseToQueryResult(t *fbproto.TableResponse) (*featurebase.SQLRespo return qr, nil } -func datatypeToType(ciDatatype string) string { +func datatypeToBaseType(ciDatatype string) dax.BaseType { switch ciDatatype { case "string": - return parser.FieldTypeString + return dax.BaseTypeString case "uint64": - return parser.FieldTypeID + return dax.BaseTypeID case "float64": // ?? panic("float64 doesn't have sql3 field type?") case "int64": - return parser.FieldTypeInt + return dax.BaseTypeInt case "bool": - return parser.FieldTypeBool + return dax.BaseTypeBool case "decimal": - return parser.FieldTypeDecimal + return dax.BaseTypeDecimal case "timestamp": - return parser.FieldTypeTimestamp + return dax.BaseTypeTimestamp case "[]string": - return parser.FieldTypeStringSet + return dax.BaseTypeStringSet case "[]uint64": - return parser.FieldTypeIDSet + return dax.BaseTypeIDSet // TODO []byte?? default: panic(fmt.Sprintf("unknown ColumnInfo Datatype: %s", ciDatatype)) diff --git a/dax/queryer/schema_api.go b/dax/queryer/schema_api.go index 27e85b82e..15ad613fe 100644 --- a/dax/queryer/schema_api.go +++ b/dax/queryer/schema_api.go @@ -93,7 +93,7 @@ func daxFieldToFeaturebaseFieldInfo(field *dax.Field) (*pilosa.FieldInfo, error) max := field.Options.Max switch field.Type { - case dax.FieldTypeTimestamp: + case dax.BaseTypeTimestamp: timestampOptions, err := daxFieldOptionsToFeaturebaseTimestamp(field.Options) if err != nil { return nil, errors.Wrap(err, "getting timestamp options") @@ -132,12 +132,12 @@ func daxFieldToFeaturebaseFieldInfo(field *dax.Field) (*pilosa.FieldInfo, error) // dax.Field. func featurebaseFieldType(f *dax.Field) string { switch f.Type { - case dax.FieldTypeID, dax.FieldTypeString: + case dax.BaseTypeID, dax.BaseTypeString: if f.Name == dax.PrimaryKeyFieldName { return string(f.Type) } return "mutex" - case dax.FieldTypeIDSet, dax.FieldTypeStringSet: + case dax.BaseTypeIDSet, dax.BaseTypeStringSet: if f.Options.TimeQuantum != "" { return "time" } @@ -189,7 +189,7 @@ func featurebaseFieldOptionsToDaxField(name string, fo *pilosa.FieldOptions) (*d // values are applied in sql3/planner/createtable.go, so we don't // initialize with defaults here. In other words, we set these value to // exactly as we receive them from the caller. - var fieldType dax.FieldType + var fieldType dax.BaseType var min pql.Decimal var max pql.Decimal var scale int64 @@ -203,41 +203,41 @@ func featurebaseFieldOptionsToDaxField(name string, fo *pilosa.FieldOptions) (*d switch fo.Type { case pilosa.FieldTypeMutex: if fo.Keys { - fieldType = dax.FieldTypeString + fieldType = dax.BaseTypeString } else { - fieldType = dax.FieldTypeID + fieldType = dax.BaseTypeID } cacheType = fo.CacheType cacheSize = fo.CacheSize case pilosa.FieldTypeSet: if fo.Keys { - fieldType = dax.FieldTypeStringSet + fieldType = dax.BaseTypeStringSet } else { - fieldType = dax.FieldTypeIDSet + fieldType = dax.BaseTypeIDSet } cacheType = fo.CacheType cacheSize = fo.CacheSize case pilosa.FieldTypeInt: min = fo.Min max = fo.Max - fieldType = dax.FieldTypeInt + fieldType = dax.BaseTypeInt foreignIndex = fo.ForeignIndex case pilosa.FieldTypeDecimal: min = fo.Min max = fo.Max scale = fo.Scale - fieldType = dax.FieldTypeDecimal + fieldType = dax.BaseTypeDecimal case pilosa.FieldTypeTimestamp: epoch = featurebaseFieldOptionsToEpoch(fo) timeUnit = fo.TimeUnit - fieldType = dax.FieldTypeTimestamp + fieldType = dax.BaseTypeTimestamp case pilosa.FieldTypeBool: - fieldType = dax.FieldTypeBool + fieldType = dax.BaseTypeBool case pilosa.FieldTypeTime: if fo.Keys { - fieldType = dax.FieldTypeStringSet + fieldType = dax.BaseTypeStringSet } else { - fieldType = dax.FieldTypeIDSet + fieldType = dax.BaseTypeIDSet } timeQuantum = dax.TimeQuantum(fo.TimeQuantum) default: @@ -316,11 +316,11 @@ func (s *qualifiedSchemaAPI) CreateIndexAndFields(ctx context.Context, indexName daxFields := make([]*dax.Field, 0, len(fields)+1) // Add the primary key field. - var fieldType dax.FieldType + var fieldType dax.BaseType if options.Keys { - fieldType = dax.FieldTypeString + fieldType = dax.BaseTypeString } else { - fieldType = dax.FieldTypeID + fieldType = dax.BaseTypeID } daxFields = append(daxFields, &dax.Field{ Name: dax.PrimaryKeyFieldName, diff --git a/dax/table.go b/dax/table.go index 09c584978..2ef28a739 100644 --- a/dax/table.go +++ b/dax/table.go @@ -59,16 +59,16 @@ const TableKeyDelimiter = "__" // which make up the TableKey) be at the beginning of the TableKey. const PrefixTable = "tbl" -// Field types. +// Base types. const ( - FieldTypeBool = "bool" // - FieldTypeDecimal = "decimal" // - FieldTypeID = "id" // non-keyed mutex - FieldTypeIDSet = "idset" // non-keyed set - FieldTypeInt = "int" // - FieldTypeString = "string" // keyed mutex - FieldTypeStringSet = "stringset" // keyed set - FieldTypeTimestamp = "timestamp" // + BaseTypeBool = "bool" // + BaseTypeDecimal = "decimal" // + BaseTypeID = "id" // non-keyed mutex + BaseTypeIDSet = "idset" // non-keyed set + BaseTypeInt = "int" // + BaseTypeString = "string" // keyed mutex + BaseTypeStringSet = "stringset" // keyed set + BaseTypeTimestamp = "timestamp" // DefaultPartitionN = 256 @@ -179,7 +179,7 @@ func (t *Table) CreateID() (TableID, error) { // In order to avoid creating an ID with a double underscore, we remove all // underscores from the original table name (because that's what we use in // TableKey as a delimiter). In addition to that, we remove any other - // characters which are not valid as a pilosa indes name. + // characters which are not valid as a pilosa index name. stub := regexp.MustCompile(`[^a-z0-9-]+`).ReplaceAllString(strings.ToLower(string(t.Name)), "") if len(stub) > 10 { stub = stub[:10] @@ -208,7 +208,7 @@ func NewTable(name TableName) *Table { func (t *Table) StringKeys() bool { for _, fld := range t.Fields { if fld.IsPrimaryKey() { - if fld.Type == FieldTypeString { + if fld.Type == BaseTypeString { return true } break @@ -225,7 +225,7 @@ func (t *Table) HasValidPrimaryKey() bool { continue } - if fld.Type == FieldTypeID || fld.Type == FieldTypeString { + if fld.Type == BaseTypeID || fld.Type == BaseTypeString { return true } } @@ -491,13 +491,32 @@ func (o QualifiedTables) Swap(i, j int) { o[i], o[j] = o[j], o[i] } // FieldName is a typed string used for field names. type FieldName string -// FieldType is a typed string used for field types. -type FieldType string +// BaseType is a typed string used for field types. +type BaseType string + +// BaseTypeFromString converts a string to one of the defined BaseTypes. If the +// string does not match a BaseType, then an error is returned. +func BaseTypeFromString(s string) (BaseType, error) { + lowered := strings.ToLower(s) + switch lowered { + case BaseTypeBool, + BaseTypeDecimal, + BaseTypeID, + BaseTypeIDSet, + BaseTypeInt, + BaseTypeString, + BaseTypeStringSet, + BaseTypeTimestamp: + return BaseType(lowered), nil + default: + return "", errors.Errorf("invalid field type: %s", s) + } +} // Field represents a field and its configuration. type Field struct { Name FieldName `json:"name"` - Type FieldType `json:"type"` + Type BaseType `json:"type"` Options FieldOptions `json:"options"` } @@ -509,7 +528,7 @@ func (f *Field) String() string { // StringKeys returns true if the field uses string keys. func (f *Field) StringKeys() bool { switch f.Type { - case FieldTypeString, FieldTypeStringSet: + case BaseTypeString, BaseTypeStringSet: return true } return false @@ -539,13 +558,13 @@ func (f *Field) constraints() string { // Apply constraints. switch f.Type { - case FieldTypeInt: + case BaseTypeInt: sql += fmt.Sprintf(" MIN %d MAX %d", f.Options.Min.ToInt64(0), f.Options.Max.ToInt64(0)) - case FieldTypeID, FieldTypeString: + case BaseTypeID, BaseTypeString: if f.Options.CacheType != "" { sql += fmt.Sprintf(" CACHETYPE %s SIZE %d", f.Options.CacheType, f.Options.CacheSize) } - case FieldTypeIDSet, FieldTypeStringSet: + case BaseTypeIDSet, BaseTypeStringSet: if f.Options.CacheType != "" { sql += fmt.Sprintf(" CACHETYPE %s SIZE %d", f.Options.CacheType, f.Options.CacheSize) } @@ -555,7 +574,7 @@ func (f *Field) constraints() string { sql += fmt.Sprintf(" TTL '%s'", f.Options.TTL) } } - case FieldTypeTimestamp: + case BaseTypeTimestamp: if f.Options.TimeUnit != "" { sql += fmt.Sprintf(" TIMEUNIT '%s'", f.Options.TimeUnit) if !f.Options.Epoch.IsZero() { diff --git a/dax/table_test.go b/dax/table_test.go index 8aa5c84e7..8188c60fb 100644 --- a/dax/table_test.go +++ b/dax/table_test.go @@ -31,11 +31,11 @@ func TestTable(t *testing.T) { Fields: []*dax.Field{ { Name: dax.PrimaryKeyFieldName, - Type: dax.FieldTypeString, + Type: dax.BaseTypeString, }, { Name: "stringField2", - Type: dax.FieldTypeString, + Type: dax.BaseTypeString, }, }, } @@ -48,7 +48,7 @@ func TestTable(t *testing.T) { Fields: []*dax.Field{ { Name: dax.PrimaryKeyFieldName, - Type: dax.FieldTypeID, + Type: dax.BaseTypeID, }, }, } diff --git a/dax/test/dax/dax_test.go b/dax/test/dax/dax_test.go index 72c99ae66..258facc76 100644 --- a/dax/test/dax/dax_test.go +++ b/dax/test/dax/dax_test.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "sort" - "strconv" "strings" "testing" "time" @@ -26,8 +25,6 @@ import ( "github.com/molecula/featurebase/v3/dax/test/featurebase" "github.com/molecula/featurebase/v3/dax/test/inspector" "github.com/molecula/featurebase/v3/pql" - "github.com/molecula/featurebase/v3/sql3/parser" - planner_types "github.com/molecula/featurebase/v3/sql3/planner/types" "github.com/molecula/featurebase/v3/sql3/test/defs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -95,14 +92,7 @@ func TestDAXIntegration(t *testing.T) { // skips is a list of tests which are currently not passing in dax. We // need to get these passing before alpha. skips := []string{ - "testinsert/test-5", - "unoptestd/test-0", - "unoptestd/test-2", - "binoptesti_d/test-12", - "binoptestid_d/test-12", - "binoptestdec_i/test-12", - "binoptestdec_id/test-12", - "binoptestdec_d/test-12", + "testinsert/test-5", // error messages differ "table-82/test-3", "table-82/test-8", "table-83/test-3", @@ -179,9 +169,9 @@ func TestDAXIntegration(t *testing.T) { assert.ElementsMatch(t, sqltest.ExpHdrs, headers) // make a map of column name to header index - m := make(map[string]int) + m := make(map[dax.FieldName]int) for i := range headers { - m[headers[i].ColumnName] = i + m[dax.FieldName(headers[i].Name)] = i } // Put the expRows in the same column order as the headers returned @@ -190,7 +180,7 @@ func TestDAXIntegration(t *testing.T) { for i := range sqltest.ExpRows { exp[i] = make([]interface{}, len(headers)) for j := range sqltest.ExpHdrs { - targetIdx := m[sqltest.ExpHdrs[j].ColumnName] + targetIdx := m[sqltest.ExpHdrs[j].Name] assert.GreaterOrEqual(t, len(sqltest.ExpRows[i]), len(headers), "expected row set has fewer columns than returned headers") exp[i][targetIdx] = sqltest.ExpRows[i][j] @@ -242,9 +232,9 @@ func TestDAXIntegration(t *testing.T) { assert.ElementsMatch(t, pqltest.ExpHdrs, headers) // make a map of column name to header index - m := make(map[string]int) + m := make(map[dax.FieldName]int) for i := range headers { - m[headers[i].ColumnName] = i + m[dax.FieldName(headers[i].Name)] = i } // Put the expRows in the same column order as the headers returned @@ -253,7 +243,7 @@ func TestDAXIntegration(t *testing.T) { for i := range pqltest.ExpRows { exp[i] = make([]interface{}, len(headers)) for j := range pqltest.ExpHdrs { - targetIdx := m[pqltest.ExpHdrs[j].ColumnName] + targetIdx := m[pqltest.ExpHdrs[j].Name] assert.GreaterOrEqual(t, len(pqltest.ExpRows[i]), len(headers), "expected row set has fewer columns than returned headers") exp[i][targetIdx] = pqltest.ExpRows[i][j] @@ -604,7 +594,7 @@ func TestDAXIntegration(t *testing.T) { addressFn(qcName), qual, `select count(*) as cnt from tbl where setcontains(a_string_set, 'A90B')`, - `{"schema":{"fields":[{"name":"cnt","type":"INT"}]},"data":[[1]],"error":"","warnings":null}`, + `{"schema":{"fields":[{"name":"cnt","type":"int","base-type":"int"}]},"data":[[1]],"error":"","warnings":null}`, ) }) @@ -833,7 +823,7 @@ func TestDAXIntegration(t *testing.T) { addressFn(qcName), qual, `select count(*) as cnt from tbl where setcontains(a_string_set, 'A90B')`, - `{"schema":{"fields":[{"name":"cnt","type":"INT"}]},"data":[[1]],"error":"","warnings":null}`, + `{"schema":{"fields":[{"name":"cnt","type":"int","base-type":"int"}]},"data":[[1]],"error":"","warnings":null}`, ) }) @@ -874,7 +864,7 @@ func TestDAXIntegration(t *testing.T) { addressFn(qcName), qual, `select count(*) as cnt from tbl where setcontains(a_string_set, 'A90B')`, - `{"schema":{"fields":[{"name":"cnt","type":"INT"}]},"data":[[1]],"error":"","warnings":null}`, + `{"schema":{"fields":[{"name":"cnt","type":"int","base-type":"int"}]},"data":[[1]],"error":"","warnings":null}`, ) }) @@ -1023,7 +1013,7 @@ func TestDAXIntegration(t *testing.T) { addressFn(qcName), qual, `select count(*) as cnt from tbl where setcontains(a_string_set, 'B25A')`, - `{"schema":{"fields":[{"name":"cnt","type":"INT"}]},"data":[[1]],"error":"","warnings":null}`, + `{"schema":{"fields":[{"name":"cnt","type":"int","base-type":"int"}]},"data":[[1]],"error":"","warnings":null}`, ) }) @@ -1086,7 +1076,7 @@ func TestDAXIntegration(t *testing.T) { addressFn(qcName), qual, `select count(*) as cnt from tbl where setcontains(a_string_set, 'B25A')`, - `{"schema":{"fields":[{"name":"cnt","type":"INT"}]},"data":[[2]],"error":"","warnings":null}`, + `{"schema":{"fields":[{"name":"cnt","type":"int","base-type":"int"}]},"data":[[2]],"error":"","warnings":null}`, ) }) @@ -1128,7 +1118,7 @@ func TestDAXIntegration(t *testing.T) { addressFn(qcName), qual, `select count(*) as cnt from tbl where setcontains(a_string_set, 'B25A')`, - `{"schema":{"fields":[{"name":"cnt","type":"INT"}]},"data":[[2]],"error":"","warnings":null}`, + `{"schema":{"fields":[{"name":"cnt","type":"int","base-type":"int"}]},"data":[[2]],"error":"","warnings":null}`, ) }) @@ -1461,7 +1451,7 @@ func TestDAXIntegration(t *testing.T) { addressFn(qcName), qual, `select count(*) as cnt from tbl`, - `{"schema":{"fields":[{"name":"cnt","type":"INT"}]},"data":[[100]],"error":"","warnings":null}`, + `{"schema":{"fields":[{"name":"cnt","type":"int","base-type":"int"}]},"data":[[100]],"error":"","warnings":null}`, ) }) @@ -1481,7 +1471,7 @@ func TestDAXIntegration(t *testing.T) { addressFn(qcName), qual, `select count(*) as cnt from tbl`, - `{"schema":{"fields":[{"name":"cnt","type":"INT"}]},"data":[[0]],"error":"","warnings":null}`, + `{"schema":{"fields":[{"name":"cnt","type":"int","base-type":"int"}]},"data":[[0]],"error":"","warnings":null}`, ) }) }) @@ -1587,10 +1577,10 @@ func sqlCheck(t *testing.T, c docker.Container, address dax.Address, qual dax.Ta out := sqlRun(t, c, address, qual, sql) - got := &fb.SQLResponse{} + got := &fb.WireQueryResponse{} assert.NoError(t, json.Unmarshal([]byte(out), got)) - want := &fb.SQLResponse{} + want := &fb.WireQueryResponse{} assert.NoError(t, json.Unmarshal([]byte(exp), want)) assert.Equal(t, want.Schema, got.Schema) @@ -1666,7 +1656,6 @@ func pqlRun(tb testing.TB, c docker.Container, address dax.Address, qual dax.Tab cmd := req.Cmd() log.Printf("cmd: %s", cmd) - //resp, err := insp.ExecResp(ctx, c, req.Cmd()) resp, err := insp.ExecResp(ctx, c, cmd) assert.NoError(tb, err) @@ -1803,7 +1792,7 @@ func sortStringKeys(in [][]interface{}) { } // mustQueryRows returns the row results as a slice of []interface{}, along with the columns. -func mustQueryRows(tb testing.TB, c docker.Container, address dax.Address, qual dax.TableQualifier, query string, table string) ([][]interface{}, []*planner_types.PlannerColumn, error) { +func mustQueryRows(tb testing.TB, c docker.Container, address dax.Address, qual dax.TableQualifier, query string, table string) ([][]interface{}, []*fb.WireQueryField, error) { tb.Helper() var sql string var out string @@ -1814,105 +1803,16 @@ func mustQueryRows(tb testing.TB, c docker.Container, address dax.Address, qual out = pqlRun(tb, c, address, qual, table, query) } - sqlResp := &fb.SQLResponse{} + sqlResp := &fb.WireQueryResponse{} - err := json.Unmarshal([]byte(out), sqlResp) - if err != nil { + if err := json.Unmarshal([]byte(out), sqlResp); err != nil { tb.Fatalf("error unmarshaling response: %v, raw resp: '%s'", err, out) } - headers := make([]*planner_types.PlannerColumn, len(sqlResp.Schema.Fields)) - for i, fld := range sqlResp.Schema.Fields { - var htype parser.ExprDataType - - if strings.HasPrefix(fld.Type, parser.FieldTypeDecimal) { - sscale := fld.Type[8 : len(fld.Type)-1] - scale, err := strconv.Atoi(sscale) - assert.NoError(tb, err) - htype = &parser.DataTypeDecimal{ - Scale: int64(scale), - } - - } else { - switch fld.Type { - case parser.FieldTypeBool: - htype = &parser.DataTypeBool{} - case parser.FieldTypeID: - htype = &parser.DataTypeID{} - case parser.FieldTypeIDSet: - htype = &parser.DataTypeIDSet{} - case parser.FieldTypeIDSetQuantum: - htype = &parser.DataTypeIDSetQuantum{} - case parser.FieldTypeInt: - htype = &parser.DataTypeInt{} - case parser.FieldTypeString: - htype = &parser.DataTypeString{} - case parser.FieldTypeStringSet: - htype = &parser.DataTypeStringSet{} - case parser.FieldTypeStringSetQuantum: - htype = &parser.DataTypeStringSetQuantum{} - case parser.FieldTypeTimestamp: - htype = &parser.DataTypeTimestamp{} - default: - tb.Errorf("unsupported header type: %s", fld.Type) - } - } - - headers[i] = &planner_types.PlannerColumn{ - ColumnName: fld.Name, - Type: htype, - } - } - - data := sqlResp.Data - - // try to convert the types based on the headers - for i := range data { - for j, hdr := range headers { - switch ht := hdr.Type.(type) { - case *parser.DataTypeID, *parser.DataTypeInt: - if _, ok := data[i][j].(float64); ok { - data[i][j] = int64(data[i][j].(float64)) - } - - case *parser.DataTypeIDSet: - if src, ok := data[i][j].([]interface{}); ok { - val := make([]int64, len(src)) - for k := range src { - val[k] = int64(src[k].(float64)) - } - data[i][j] = val - } - - case *parser.DataTypeDecimal: - if _, ok := data[i][j].(float64); ok { - format := fmt.Sprintf("%%.%df", ht.Scale) - dec, err := pql.ParseDecimal(fmt.Sprintf(format, data[i][j])) - assert.NoError(tb, err) - data[i][j] = dec - } - - case *parser.DataTypeStringSet: - if src, ok := data[i][j].([]interface{}); ok { - val := make([]string, len(src)) - for k := range src { - val[k] = src[k].(string) - } - data[i][j] = val - } - - case *parser.DataTypeBool, *parser.DataTypeString: - // no need to convert - - default: - log.Printf("WARNING: unimplemented: %T", ht) - } - } - } - + var errOut error if sqlResp.Error != "" { - err = errors.New(sqlResp.Error) + errOut = errors.New(sqlResp.Error) } - return data, headers, err + return sqlResp.Data, sqlResp.Schema.Fields, errOut } diff --git a/dax/test/table.go b/dax/test/table.go index 3c400c127..05431bb74 100644 --- a/dax/test/table.go +++ b/dax/test/table.go @@ -14,11 +14,11 @@ import ( func TestQualifiedTable(t *testing.T, qual dax.TableQualifier, name dax.TableName, partitionN int, keyed bool) *dax.QualifiedTable { t.Helper() - var pkFieldType dax.FieldType + var pkFieldType dax.BaseType if keyed { - pkFieldType = dax.FieldTypeString + pkFieldType = dax.BaseTypeString } else { - pkFieldType = dax.FieldTypeID + pkFieldType = dax.BaseTypeID } tbl := dax.NewTable(name) @@ -42,11 +42,11 @@ func TestQualifiedTable(t *testing.T, qual dax.TableQualifier, name dax.TableNam func TestQualifiedTableWithID(t *testing.T, qual dax.TableQualifier, id string, name dax.TableName, partitionN int, keyed bool) *dax.QualifiedTable { t.Helper() - var pkFieldType dax.FieldType + var pkFieldType dax.BaseType if keyed { - pkFieldType = dax.FieldTypeString + pkFieldType = dax.BaseTypeString } else { - pkFieldType = dax.FieldTypeID + pkFieldType = dax.BaseTypeID } tbl := &dax.Table{ diff --git a/fbcloud/client.go b/fbcloud/client.go index fbb9e0233..c675763d5 100644 --- a/fbcloud/client.go +++ b/fbcloud/client.go @@ -47,7 +47,7 @@ type tokenizedSQL struct { } // Query issues a SQL query formatted for the FeatureBase cloud query endpoint. -func (cq *Queryer) Query(org, db, sql string) (*featurebase.SQLResponse, error) { +func (cq *Queryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) { if time.Since(cq.lastRefresh) > TokenRefreshTimeout { if err := cq.tokenRefresh(); err != nil { return nil, errors.Wrap(err, "refreshing token") @@ -143,5 +143,5 @@ func (cq *Queryer) HTTPRequest(method, path, body string, v interface{}) ([]byte } type cloudResponse struct { - Results featurebase.SQLResponse `json:"results"` + Results featurebase.WireQueryResponse `json:"results"` } diff --git a/http_handler.go b/http_handler.go index a99feed80..522f08bc6 100644 --- a/http_handler.go +++ b/http_handler.go @@ -951,7 +951,7 @@ type successResponse struct { Error *HTTPError `json:"error,omitempty"` } -// Error defines a standard application error. +// HTTPError defines a standard application error. type HTTPError struct { // Human-readable message. Message string `json:"message"` @@ -1491,13 +1491,21 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { // Read schema & write to response. columns := rootOperator.Schema() - schema := SQLSchema{ - Fields: make([]*SQLField, len(columns)), + schema := WireQuerySchema{ + Fields: make([]*WireQueryField, len(columns)), } for i, col := range columns { - schema.Fields[i] = &SQLField{ - Name: col.ColumnName, - Type: col.Type.TypeName(), + btype, err := dax.BaseTypeFromString(col.Type.TypeName()) + if err != nil { + writeError(err) + writeWarnings(rootOperator.Warnings()) + return + } + schema.Fields[i] = &WireQueryField{ + Name: dax.FieldName(col.ColumnName), + Type: strings.ToLower(col.Type.TypeDescription()), // TODO(tlt): remove this once sql3 uses BaseTypes. + BaseType: btype, + TypeInfo: col.Type.TypeInfo(), } } w.Write([]byte(`"schema":`)) @@ -1547,15 +1555,6 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { writePlan(rootOperator.Plan()) } -type SQLField struct { - Name string `json:"name"` - Type string `json:"type"` -} - -type SQLSchema struct { - Fields []*SQLField `json:"fields"` -} - func (h *Handler) handleCPUProfileStart(w http.ResponseWriter, r *http.Request) { if h.pprofCPUProfileBuffer == nil { h.pprofCPUProfileBuffer = bytes.NewBuffer(nil) diff --git a/idk/ingest_test.go b/idk/ingest_test.go index 53dd759cc..651f2813f 100644 --- a/idk/ingest_test.go +++ b/idk/ingest_test.go @@ -1757,7 +1757,7 @@ func TestBatchTargetMDS(t *testing.T) { t.Run("FieldTypes", func(t *testing.T) { tests := []struct { - fieldType dax.FieldType + fieldType dax.BaseType fieldOptions dax.FieldOptions fieldFn fieldFn in [][]interface{} @@ -1771,7 +1771,7 @@ func TestBatchTargetMDS(t *testing.T) { // }, // }, { - fieldType: dax.FieldTypeDecimal, + fieldType: dax.BaseTypeDecimal, fieldOptions: dax.FieldOptions{ Scale: 4, }, @@ -1782,7 +1782,7 @@ func TestBatchTargetMDS(t *testing.T) { }, }, { - fieldType: dax.FieldTypeID, + fieldType: dax.BaseTypeID, fieldFn: idFn, in: [][]interface{}{ {1, uint64(11)}, @@ -1790,7 +1790,7 @@ func TestBatchTargetMDS(t *testing.T) { }, }, { - fieldType: dax.FieldTypeIDSet, + fieldType: dax.BaseTypeIDSet, fieldFn: idSetFn, in: [][]interface{}{ {1, []uint64{11, 12, 13}}, @@ -1798,7 +1798,7 @@ func TestBatchTargetMDS(t *testing.T) { }, }, { - fieldType: dax.FieldTypeInt, + fieldType: dax.BaseTypeInt, fieldFn: intFn, in: [][]interface{}{ {1, int(11)}, @@ -1807,7 +1807,7 @@ func TestBatchTargetMDS(t *testing.T) { fieldOptions: dax.FieldOptions{Min: pql.NewDecimal(-100, 0), Max: pql.NewDecimal(100, 0)}, }, { - fieldType: dax.FieldTypeString, + fieldType: dax.BaseTypeString, fieldFn: stringFn, in: [][]interface{}{ {1, "cycling"}, @@ -1815,7 +1815,7 @@ func TestBatchTargetMDS(t *testing.T) { }, }, { - fieldType: dax.FieldTypeStringSet, + fieldType: dax.BaseTypeStringSet, fieldFn: stringSetFn, in: [][]interface{}{ {1, []string{"cycling", "swimming"}}, @@ -1823,7 +1823,7 @@ func TestBatchTargetMDS(t *testing.T) { }, }, { - fieldType: dax.FieldTypeTimestamp, + fieldType: dax.BaseTypeTimestamp, fieldFn: timestampFn, in: [][]interface{}{ {1, time.Now()}, @@ -1846,7 +1846,7 @@ func TestBatchTargetMDS(t *testing.T) { tbl.Fields = []*dax.Field{ { Name: dax.PrimaryKeyFieldName, - Type: dax.FieldTypeID, + Type: dax.BaseTypeID, }, { Name: dax.FieldName(fieldName), diff --git a/idk/mds/schemamanager.go b/idk/mds/schemamanager.go index 22ba54ef9..21f93f2ec 100644 --- a/idk/mds/schemamanager.go +++ b/idk/mds/schemamanager.go @@ -49,28 +49,28 @@ func (s *schemaManager) Schema() (*featurebase_client.Schema, error) { opts := make([]featurebase_client.FieldOption, 0) switch fld.Type { - case dax.FieldTypeBool: + case dax.BaseTypeBool: opts = append(opts, featurebase_client.OptFieldTypeBool()) - case dax.FieldTypeDecimal: + case dax.BaseTypeDecimal: opts = append(opts, featurebase_client.OptFieldTypeDecimal( fld.Options.Scale, )) - case dax.FieldTypeID: + case dax.BaseTypeID: opts = append(opts, featurebase_client.OptFieldTypeMutex( featurebase_client.CacheType(fld.Options.CacheType), int(fld.Options.CacheSize), )) - case dax.FieldTypeIDSet: + case dax.BaseTypeIDSet: opts = append(opts, featurebase_client.OptFieldTypeSet( featurebase_client.CacheType(fld.Options.CacheType), int(fld.Options.CacheSize), )) - case dax.FieldTypeInt: + case dax.BaseTypeInt: opts = append(opts, featurebase_client.OptFieldTypeInt( fld.Options.Min.ToInt64(0), fld.Options.Max.ToInt64(0), )) - case dax.FieldTypeString: + case dax.BaseTypeString: opts = append(opts, featurebase_client.OptFieldTypeMutex( featurebase_client.CacheType(fld.Options.CacheType), @@ -78,7 +78,7 @@ func (s *schemaManager) Schema() (*featurebase_client.Schema, error) { ), featurebase_client.OptFieldKeys(true), ) - case dax.FieldTypeStringSet: + case dax.BaseTypeStringSet: opts = append(opts, featurebase_client.OptFieldTypeSet( featurebase_client.CacheType(fld.Options.CacheType), @@ -86,7 +86,7 @@ func (s *schemaManager) Schema() (*featurebase_client.Schema, error) { ), featurebase_client.OptFieldKeys(true), ) - case dax.FieldTypeTimestamp: + case dax.BaseTypeTimestamp: opts = append(opts, featurebase_client.OptFieldTypeTimestamp( featurebase_client.DefaultEpoch, fld.Options.TimeUnit, diff --git a/qa/testcases/smoketest/test_smoke.py b/qa/testcases/smoketest/test_smoke.py index acea92c5c..a6ff8245b 100644 --- a/qa/testcases/smoketest/test_smoke.py +++ b/qa/testcases/smoketest/test_smoke.py @@ -44,7 +44,8 @@ def test_sql3_is_responding(): assert response.status_code == 200 assert response.headers["Content-Type"] == "application/json" resp_body = response.json() - assert resp_body['schema']['fields'][0]['type'] == "INT" + assert resp_body['schema']['fields'][0]['type'] == "int" + assert resp_body['schema']['fields'][0]['base-type'] == "int" assert resp_body['data'][0][0] == 1 def test_get_index_api(): @@ -66,4 +67,4 @@ def test_set_and_read_query_api(): assert response.status_code == 200 assert response.headers["Content-Type"] == "application/json" resp_body = response.json() - assert resp_body['results'][0]['columns'][0] == 10 \ No newline at end of file + assert resp_body['results'][0]['columns'][0] == 10 diff --git a/sql.go b/sql.go deleted file mode 100644 index db248bf64..000000000 --- a/sql.go +++ /dev/null @@ -1,9 +0,0 @@ -package pilosa - -type SQLResponse struct { - Schema SQLSchema `json:"schema"` - Data [][]interface{} `json:"data"` - Error string `json:"error"` - Warnings []string `json:"warnings"` - ExecutionTime int64 `json:"execution-time"` -} diff --git a/sql3/errors.go b/sql3/errors.go index 4f4eeee85..03b31c14d 100644 --- a/sql3/errors.go +++ b/sql3/errors.go @@ -307,7 +307,7 @@ func NewErrTypeIncompatibleWithBetweenOperator(line, col int, operator, type1 st func NewErrTypeCannotBeUsedAsRangeSubscript(line, col int, type1 string) error { return errors.New( ErrTypeCannotBeUsedAsRangeSubscript, - fmt.Sprintf("[%d:%d] type '%s' cannot be used a range subscript", line, col, type1), + fmt.Sprintf("[%d:%d] type '%s' cannot be used as a range subscript", line, col, type1), ) } diff --git a/sql3/parser/astdatatype.go b/sql3/parser/astdatatype.go index e3d980f10..f956ba09d 100644 --- a/sql3/parser/astdatatype.go +++ b/sql3/parser/astdatatype.go @@ -41,6 +41,8 @@ func IsValidTypeName(typeName string) bool { type ExprDataType interface { exprDataType() TypeName() string + TypeDescription() string + TypeInfo() map[string]interface{} } func (*DataTypeVoid) exprDataType() {} @@ -69,6 +71,14 @@ func (*DataTypeVoid) TypeName() string { return "VOID" } +func (dt *DataTypeVoid) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeVoid) TypeInfo() map[string]interface{} { + return nil +} + type DataTypeRange struct { SubscriptType ExprDataType } @@ -83,6 +93,14 @@ func (dt *DataTypeRange) TypeName() string { return fmt.Sprintf("RANGE(%s)", dt.SubscriptType.TypeName()) } +func (dt *DataTypeRange) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeRange) TypeInfo() map[string]interface{} { + return nil +} + type DataTypeTuple struct { Members []ExprDataType } @@ -104,6 +122,14 @@ func (dt *DataTypeTuple) TypeName() string { return fmt.Sprintf("TUPLE(%s)", ms) } +func (dt *DataTypeTuple) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeTuple) TypeInfo() map[string]interface{} { + return nil +} + type SubtableColumn struct { Name string DataType ExprDataType @@ -130,6 +156,14 @@ func (dt *DataTypeSubtable) TypeName() string { return fmt.Sprintf("SUBTABLE(%s)", ms) } +func (dt *DataTypeSubtable) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeSubtable) TypeInfo() map[string]interface{} { + return nil +} + type DataTypeBool struct { } @@ -141,6 +175,14 @@ func (*DataTypeBool) TypeName() string { return FieldTypeBool } +func (dt *DataTypeBool) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeBool) TypeInfo() map[string]interface{} { + return nil +} + type DataTypeDecimal struct { Scale int64 } @@ -152,9 +194,19 @@ func NewDataTypeDecimal(scale int64) *DataTypeDecimal { } func (d *DataTypeDecimal) TypeName() string { + return FieldTypeDecimal +} + +func (d *DataTypeDecimal) TypeDescription() string { return fmt.Sprintf("%s(%d)", FieldTypeDecimal, d.Scale) } +func (d *DataTypeDecimal) TypeInfo() map[string]interface{} { + return map[string]interface{}{ + "scale": d.Scale, + } +} + type DataTypeID struct { } @@ -166,6 +218,14 @@ func (*DataTypeID) TypeName() string { return FieldTypeID } +func (dt *DataTypeID) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeID) TypeInfo() map[string]interface{} { + return nil +} + type DataTypeIDSet struct { } @@ -177,6 +237,14 @@ func (*DataTypeIDSet) TypeName() string { return FieldTypeIDSet } +func (dt *DataTypeIDSet) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeIDSet) TypeInfo() map[string]interface{} { + return nil +} + type DataTypeIDSetQuantum struct { } @@ -188,6 +256,14 @@ func (*DataTypeIDSetQuantum) TypeName() string { return FieldTypeIDSetQuantum } +func (dt *DataTypeIDSetQuantum) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeIDSetQuantum) TypeInfo() map[string]interface{} { + return nil +} + type DataTypeInt struct { } @@ -199,6 +275,14 @@ func (*DataTypeInt) TypeName() string { return FieldTypeInt } +func (dt *DataTypeInt) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeInt) TypeInfo() map[string]interface{} { + return nil +} + type DataTypeString struct { } @@ -210,6 +294,14 @@ func (*DataTypeString) TypeName() string { return FieldTypeString } +func (dt *DataTypeString) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeString) TypeInfo() map[string]interface{} { + return nil +} + type DataTypeStringSet struct { } @@ -221,6 +313,14 @@ func (*DataTypeStringSet) TypeName() string { return FieldTypeStringSet } +func (dt *DataTypeStringSet) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeStringSet) TypeInfo() map[string]interface{} { + return nil +} + type DataTypeStringSetQuantum struct { } @@ -232,6 +332,14 @@ func (*DataTypeStringSetQuantum) TypeName() string { return FieldTypeStringSetQuantum } +func (dt *DataTypeStringSetQuantum) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeStringSetQuantum) TypeInfo() map[string]interface{} { + return nil +} + type DataTypeTimestamp struct { } @@ -243,6 +351,14 @@ func (*DataTypeTimestamp) TypeName() string { return FieldTypeTimestamp } +func (dt *DataTypeTimestamp) TypeDescription() string { + return dt.TypeName() +} + +func (*DataTypeTimestamp) TypeInfo() map[string]interface{} { + return nil +} + func StringToDecimal(v string) (pql.Decimal, error) { fvalue, err := strconv.ParseFloat(v, 64) if err != nil { diff --git a/sql3/planner/expression.go b/sql3/planner/expression.go index 7be56a3ed..c9a04e14e 100644 --- a/sql3/planner/expression.go +++ b/sql3/planner/expression.go @@ -2060,7 +2060,7 @@ func (n *castPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er i, err := strconv.Atoi(nl) if err != nil { //TODO(pok) need to push location into here - return nil, sql3.NewErrInvalidCast(0, 0, nl, n.targetType.TypeName()) + return nil, sql3.NewErrInvalidCast(0, 0, nl, n.targetType.TypeDescription()) } return int64(i), nil @@ -2068,7 +2068,7 @@ func (n *castPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er i, err := strconv.ParseBool(nl) if err != nil { //TODO(pok) need to push location into here - return nil, sql3.NewErrInvalidCast(0, 0, nl, n.targetType.TypeName()) + return nil, sql3.NewErrInvalidCast(0, 0, nl, n.targetType.TypeDescription()) } return i, nil @@ -2076,13 +2076,13 @@ func (n *castPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er fvalue, err := strconv.ParseFloat(nl, 64) if err != nil { //TODO(pok) need to push location into here - return nil, sql3.NewErrInvalidCast(0, 0, nl, n.targetType.TypeName()) + return nil, sql3.NewErrInvalidCast(0, 0, nl, n.targetType.TypeDescription()) } scale := parser.NumDecimalPlaces(nl) unscaledValue := int64(fvalue * math.Pow(10, float64(scale))) castValue := pql.NewDecimal(unscaledValue, int64(scale)) if tt.Scale < castValue.Scale { - return nil, sql3.NewErrInvalidCast(0, 0, nl, n.targetType.TypeName()) + return nil, sql3.NewErrInvalidCast(0, 0, nl, n.targetType.TypeDescription()) } return castValue, nil @@ -2098,7 +2098,7 @@ func (n *castPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er } else if tm, err := time.ParseInLocation("2006-01-02", nl, time.UTC); err == nil { return tm, nil } else { - return nil, sql3.NewErrInvalidCast(0, 0, nl, n.targetType.TypeName()) + return nil, sql3.NewErrInvalidCast(0, 0, nl, n.targetType.TypeDescription()) } } diff --git a/sql3/planner/expressionanalyzer.go b/sql3/planner/expressionanalyzer.go index 10efcb646..98a4f0a0f 100644 --- a/sql3/planner/expressionanalyzer.go +++ b/sql3/planner/expressionanalyzer.go @@ -36,7 +36,7 @@ func (p *ExecutionPlanner) analyzeExpression(expr parser.Expr, scope parser.Stat return nil, err } if !typesCanBeCast(analyzedExpr.DataType(), targetType) { - return nil, sql3.NewErrInvalidCast(analyzedExpr.Pos().Line, analyzedExpr.Pos().Column, analyzedExpr.DataType().TypeName(), targetType.TypeName()) + return nil, sql3.NewErrInvalidCast(analyzedExpr.Pos().Line, analyzedExpr.Pos().Column, analyzedExpr.DataType().TypeDescription(), targetType.TypeDescription()) } e.X = analyzedExpr e.ResultDataType = targetType @@ -256,7 +256,7 @@ func (p *ExecutionPlanner) analyzeExpression(expr parser.Expr, scope parser.Stat //we are "case expr when" form, so need to make sure that 'expr' and all block conditions are equatable for _, blk := range e.Blocks { if !typesAreComparable(e.Operand.DataType(), blk.Condition.DataType()) { - return nil, sql3.NewErrTypesAreNotEquatable(blk.Condition.Pos().Line, blk.Condition.Pos().Column, e.Operand.DataType().TypeName(), blk.Condition.DataType().TypeName()) + return nil, sql3.NewErrTypesAreNotEquatable(blk.Condition.Pos().Line, blk.Condition.Pos().Column, e.Operand.DataType().TypeDescription(), blk.Condition.DataType().TypeDescription()) } } } else { @@ -325,7 +325,7 @@ func (p *ExecutionPlanner) analyzeUnaryExpression(expr *parser.UnaryExpr, scope //bitwise operators case parser.BITNOT: if !typeIsCompatibleWithBitwiseOperator(x.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithBitwiseOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithBitwiseOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeDescription()) } expr.ResultDataType = x.DataType() return expr, nil @@ -333,7 +333,7 @@ func (p *ExecutionPlanner) analyzeUnaryExpression(expr *parser.UnaryExpr, scope //arithmetic operators case parser.PLUS, parser.MINUS: if !typeIsCompatibleWithArithmeticOperator(x.DataType(), op) { - return nil, sql3.NewErrTypeIncompatibleWithArithmeticOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithArithmeticOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeDescription()) } if typeIsInteger(x.DataType()) { expr.ResultDataType = parser.NewDataTypeInt() @@ -373,10 +373,10 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop //logical operators case parser.AND, parser.OR: if !typeIsCompatibleWithLogicalOperator(x.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithLogicalOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithLogicalOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeDescription()) } if !typeIsCompatibleWithLogicalOperator(y.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithLogicalOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithLogicalOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeDescription()) } //logical operator so type of expr is bool expr.ResultDataType = parser.NewDataTypeBool() @@ -385,13 +385,13 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop //equality operators case parser.EQ, parser.NE: if !typeIsCompatibleWithEqualityOperator(x.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithEqualityOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithEqualityOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeDescription()) } if !typeIsCompatibleWithEqualityOperator(y.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithEqualityOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithEqualityOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeDescription()) } if !typesAreComparable(x.DataType(), y.DataType()) { - return nil, sql3.NewErrTypesAreNotEquatable(x.Pos().Line, x.Pos().Column, x.DataType().TypeName(), y.DataType().TypeName()) + return nil, sql3.NewErrTypesAreNotEquatable(x.Pos().Line, x.Pos().Column, x.DataType().TypeDescription(), y.DataType().TypeDescription()) } //equality operator so type of expr is bool expr.ResultDataType = parser.NewDataTypeBool() @@ -400,7 +400,7 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop //comparison operators case parser.LT, parser.LE, parser.GT, parser.GE: if !typeIsCompatibleWithComparisonOperator(x.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithComparisonOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithComparisonOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeDescription()) } if typeIsTimestamp(x.DataType()) && y.IsLiteral() && typeIsString(y.DataType()) { // we have a string literal on the rhs being compared to a date so @@ -416,10 +416,10 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop } } if !typeIsCompatibleWithComparisonOperator(y.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithComparisonOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithComparisonOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeDescription()) } if !typesAreComparable(x.DataType(), y.DataType()) { - return nil, sql3.NewErrTypesAreNotEquatable(x.Pos().Line, x.Pos().Column, x.DataType().TypeName(), y.DataType().TypeName()) + return nil, sql3.NewErrTypesAreNotEquatable(x.Pos().Line, x.Pos().Column, x.DataType().TypeDescription(), y.DataType().TypeDescription()) } //comparison operator so type of expr is bool expr.ResultDataType = parser.NewDataTypeBool() @@ -428,10 +428,10 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop //arithmetic operators case parser.PLUS, parser.MINUS, parser.STAR, parser.SLASH, parser.REM: if !typeIsCompatibleWithArithmeticOperator(x.DataType(), op) { - return nil, sql3.NewErrTypeIncompatibleWithArithmeticOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithArithmeticOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeDescription()) } if !typeIsCompatibleWithArithmeticOperator(y.DataType(), op) { - return nil, sql3.NewErrTypeIncompatibleWithArithmeticOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithArithmeticOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeDescription()) } coercedType, err := typesCoercedForArithmeticOperator(x.DataType(), y.DataType(), x.Pos()) @@ -487,10 +487,10 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop //bitwise operators case parser.BITAND, parser.BITOR, parser.LSHIFT, parser.RSHIFT: if !typeIsCompatibleWithBitwiseOperator(x.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithBitwiseOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithBitwiseOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeDescription()) } if !typeIsCompatibleWithBitwiseOperator(y.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithBitwiseOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithBitwiseOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeDescription()) } coercedType, err := typesCoercedForBitwiseOperator(x.DataType(), y.DataType(), x.Pos()) if err != nil { @@ -573,7 +573,7 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop return nil, sql3.NewErrInternalf("select used as part of IN expression should only return one column") } if !typesAreComparable(x.DataType(), sel.Columns[0].Expr.DataType()) { - return nil, sql3.NewErrTypesAreNotEquatable(x.Pos().Line, x.Pos().Column, x.DataType().TypeName(), ex.DataType().TypeName()) + return nil, sql3.NewErrTypesAreNotEquatable(x.Pos().Line, x.Pos().Column, x.DataType().TypeDescription(), ex.DataType().TypeDescription()) } //need to turn this into an inner join @@ -634,7 +634,7 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop //make sure LHS and RHS types are comparable if !typesAreComparable(x.DataType(), ex.DataType()) { - return nil, sql3.NewErrTypesAreNotEquatable(x.Pos().Line, x.Pos().Column, x.DataType().TypeName(), ex.DataType().TypeName()) + return nil, sql3.NewErrTypesAreNotEquatable(x.Pos().Line, x.Pos().Column, x.DataType().TypeDescription(), ex.DataType().TypeDescription()) } } @@ -643,7 +643,7 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop case parser.BETWEEN, parser.NOTBETWEEN: if !typeIsRange(y.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithBetweenOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithBetweenOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeDescription()) } ok, err := typesAreRangeComparable(x.DataType(), y.DataType()) @@ -651,27 +651,27 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop return nil, err } if !ok { - return nil, sql3.NewErrTypeIncompatibleWithBetweenOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithBetweenOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeDescription()) } expr.ResultDataType = parser.NewDataTypeBool() return expr, nil case parser.CONCAT: if !typeIsCompatibleWithConcatOperator(x.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithConcatOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithConcatOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeDescription()) } if !typeIsCompatibleWithConcatOperator(y.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithConcatOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithConcatOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeDescription()) } expr.ResultDataType = parser.NewDataTypeString() return expr, nil case parser.LIKE, parser.NOTLIKE: if !typeIsCompatibleWithLikeOperator(x.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithLikeOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithLikeOperator(x.Pos().Line, x.Pos().Column, op.String(), x.DataType().TypeDescription()) } if !typeIsCompatibleWithLikeOperator(y.DataType()) { - return nil, sql3.NewErrTypeIncompatibleWithLikeOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeName()) + return nil, sql3.NewErrTypeIncompatibleWithLikeOperator(y.Pos().Line, y.Pos().Column, op.String(), y.DataType().TypeDescription()) } //comparison operator so type of expr is bool expr.ResultDataType = parser.NewDataTypeBool() @@ -713,10 +713,10 @@ func (p *ExecutionPlanner) analyzeRangeExpression(expr *parser.Range, scope pars } if !typeCanBeUsedInRange(expr.X.DataType()) { - return nil, sql3.NewErrTypeCannotBeUsedAsRangeSubscript(expr.X.Pos().Line, expr.X.Pos().Column, expr.X.DataType().TypeName()) + return nil, sql3.NewErrTypeCannotBeUsedAsRangeSubscript(expr.X.Pos().Line, expr.X.Pos().Column, expr.X.DataType().TypeDescription()) } if !typeCanBeUsedInRange(expr.Y.DataType()) { - return nil, sql3.NewErrTypeCannotBeUsedAsRangeSubscript(expr.Y.Pos().Line, expr.Y.Pos().Column, expr.Y.DataType().TypeName()) + return nil, sql3.NewErrTypeCannotBeUsedAsRangeSubscript(expr.Y.Pos().Line, expr.Y.Pos().Column, expr.Y.DataType().TypeDescription()) } if !typesOfRangeBoundsAreTheSame(expr.X.DataType(), expr.Y.DataType()) { return nil, sql3.NewErrIncompatibleTypesForRangeSubscripts(expr.Pos().Line, expr.Pos().Column, expr.X.DataType().TypeName(), expr.Y.DataType().TypeName()) diff --git a/sql3/planner/expressionanalyzercall.go b/sql3/planner/expressionanalyzercall.go index 8cc2aeb52..65470bf23 100644 --- a/sql3/planner/expressionanalyzercall.go +++ b/sql3/planner/expressionanalyzercall.go @@ -183,7 +183,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser } if !typesAreComparable(baseType, call.Args[1].DataType()) { - return nil, sql3.NewErrTypesAreNotEquatable(call.Args[1].Pos().Line, call.Args[1].Pos().Column, call.Args[0].DataType().TypeName(), call.Args[1].DataType().TypeName()) + return nil, sql3.NewErrTypesAreNotEquatable(call.Args[1].Pos().Line, call.Args[1].Pos().Column, call.Args[0].DataType().TypeDescription(), call.Args[1].DataType().TypeDescription()) } call.ResultDataType = parser.NewDataTypeBool() @@ -207,7 +207,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser //types from both set should be comparable if !typesAreComparable(baseType1, baseType2) { - return nil, sql3.NewErrTypesAreNotEquatable(call.Args[1].Pos().Line, call.Args[1].Pos().Column, baseType1.TypeName(), baseType2.TypeName()) + return nil, sql3.NewErrTypesAreNotEquatable(call.Args[1].Pos().Line, call.Args[1].Pos().Column, baseType1.TypeDescription(), baseType2.TypeDescription()) } call.ResultDataType = parser.NewDataTypeBool() @@ -231,7 +231,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser // types from both sets should be comparable if !typesAreComparable(baseType1, baseType2) { - return nil, sql3.NewErrTypesAreNotEquatable(call.Args[1].Pos().Line, call.Args[1].Pos().Column, baseType1.TypeName(), baseType2.TypeName()) + return nil, sql3.NewErrTypesAreNotEquatable(call.Args[1].Pos().Line, call.Args[1].Pos().Column, baseType1.TypeDescription(), baseType2.TypeDescription()) } call.ResultDataType = parser.NewDataTypeBool() diff --git a/sql3/sql_complex_test.go b/sql3/sql_complex_test.go index fd32acca7..e0fdaffb2 100644 --- a/sql3/sql_complex_test.go +++ b/sql3/sql_complex_test.go @@ -11,9 +11,9 @@ import ( "github.com/google/go-cmp/cmp" pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/dax" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/sql3/parser" - planner_types "github.com/molecula/featurebase/v3/sql3/planner/types" sql_test "github.com/molecula/featurebase/v3/sql3/test" "github.com/molecula/featurebase/v3/test" "github.com/stretchr/testify/assert" @@ -71,15 +71,15 @@ func TestPlanner_Show(t *testing.T) { t.Fatal(fmt.Errorf("unexpected result set length")) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "name", Type: parser.NewDataTypeString()}, - {ColumnName: "platform", Type: parser.NewDataTypeString()}, - {ColumnName: "platform_version", Type: parser.NewDataTypeString()}, - {ColumnName: "db_version", Type: parser.NewDataTypeString()}, - {ColumnName: "state", Type: parser.NewDataTypeString()}, - {ColumnName: "node_count", Type: parser.NewDataTypeInt()}, - {ColumnName: "shard_width", Type: parser.NewDataTypeInt()}, - {ColumnName: "replica_count", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldString("name"), + wireQueryFieldString("platform"), + wireQueryFieldString("platform_version"), + wireQueryFieldString("db_version"), + wireQueryFieldString("state"), + wireQueryFieldInt("node_count"), + wireQueryFieldInt("shard_width"), + wireQueryFieldInt("replica_count"), }, columns); diff != "" { t.Fatal(diff) } @@ -94,12 +94,12 @@ func TestPlanner_Show(t *testing.T) { t.Fatal(fmt.Errorf("unexpected result set length")) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "name", Type: parser.NewDataTypeString()}, - {ColumnName: "created_at", Type: parser.NewDataTypeTimestamp()}, - {ColumnName: "track_existence", Type: parser.NewDataTypeBool()}, - {ColumnName: "keys", Type: parser.NewDataTypeBool()}, - {ColumnName: "shard_width", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldString("name"), + wireQueryFieldTimestamp("created_at"), + wireQueryFieldBool("track_existence"), + wireQueryFieldBool("keys"), + wireQueryFieldInt("shard_width"), }, columns); diff != "" { t.Fatal(diff) } @@ -114,21 +114,21 @@ func TestPlanner_Show(t *testing.T) { t.Fatal(fmt.Errorf("unexpected result set length: %d", len(results))) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "name", Type: parser.NewDataTypeString()}, - {ColumnName: "type", Type: parser.NewDataTypeString()}, - {ColumnName: "internal_type", Type: parser.NewDataTypeString()}, - {ColumnName: "created_at", Type: parser.NewDataTypeTimestamp()}, - {ColumnName: "keys", Type: parser.NewDataTypeBool()}, - {ColumnName: "cache_type", Type: parser.NewDataTypeString()}, - {ColumnName: "cache_size", Type: parser.NewDataTypeInt()}, - {ColumnName: "scale", Type: parser.NewDataTypeInt()}, - {ColumnName: "min", Type: parser.NewDataTypeInt()}, - {ColumnName: "max", Type: parser.NewDataTypeInt()}, - {ColumnName: "timeunit", Type: parser.NewDataTypeString()}, - {ColumnName: "epoch", Type: parser.NewDataTypeInt()}, - {ColumnName: "timequantum", Type: parser.NewDataTypeString()}, - {ColumnName: "ttl", Type: parser.NewDataTypeString()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldString("name"), + wireQueryFieldString("type"), + wireQueryFieldString("internal_type"), + wireQueryFieldTimestamp("created_at"), + wireQueryFieldBool("keys"), + wireQueryFieldString("cache_type"), + wireQueryFieldInt("cache_size"), + wireQueryFieldInt("scale"), + wireQueryFieldInt("min"), + wireQueryFieldInt("max"), + wireQueryFieldString("timeunit"), + wireQueryFieldInt("epoch"), + wireQueryFieldString("timequantum"), + wireQueryFieldString("ttl"), }, columns); diff != "" { t.Fatal(diff) } @@ -143,21 +143,21 @@ func TestPlanner_Show(t *testing.T) { t.Fatal(fmt.Errorf("unexpected result set length")) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "name", Type: parser.NewDataTypeString()}, - {ColumnName: "type", Type: parser.NewDataTypeString()}, - {ColumnName: "internal_type", Type: parser.NewDataTypeString()}, - {ColumnName: "created_at", Type: parser.NewDataTypeTimestamp()}, - {ColumnName: "keys", Type: parser.NewDataTypeBool()}, - {ColumnName: "cache_type", Type: parser.NewDataTypeString()}, - {ColumnName: "cache_size", Type: parser.NewDataTypeInt()}, - {ColumnName: "scale", Type: parser.NewDataTypeInt()}, - {ColumnName: "min", Type: parser.NewDataTypeInt()}, - {ColumnName: "max", Type: parser.NewDataTypeInt()}, - {ColumnName: "timeunit", Type: parser.NewDataTypeString()}, - {ColumnName: "epoch", Type: parser.NewDataTypeInt()}, - {ColumnName: "timequantum", Type: parser.NewDataTypeString()}, - {ColumnName: "ttl", Type: parser.NewDataTypeString()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldString("name"), + wireQueryFieldString("type"), + wireQueryFieldString("internal_type"), + wireQueryFieldTimestamp("created_at"), + wireQueryFieldBool("keys"), + wireQueryFieldString("cache_type"), + wireQueryFieldInt("cache_size"), + wireQueryFieldInt("scale"), + wireQueryFieldInt("min"), + wireQueryFieldInt("max"), + wireQueryFieldString("timeunit"), + wireQueryFieldInt("epoch"), + wireQueryFieldString("timequantum"), + wireQueryFieldString("ttl"), }, columns); diff != "" { t.Fatal(diff) } @@ -383,7 +383,7 @@ func TestPlanner_CoverCreateTable(t *testing.T) { results, columns, err := sql_test.MustQueryRows(t, server, sql) assert.NoError(t, err) assert.Equal(t, [][]interface{}{}, results) - assert.Equal(t, []*planner_types.PlannerColumn{}, columns) + assert.Equal(t, []*pilosa.WireQueryField{}, columns) // Ensure that the fields got created as expected. t.Run("EnsureFields", func(t *testing.T) { @@ -460,7 +460,7 @@ func TestPlanner_CreateTable(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{}, columns); diff != "" { + if diff := cmp.Diff([]*pilosa.WireQueryField{}, columns); diff != "" { t.Fatal(diff) } }) @@ -513,7 +513,7 @@ func TestPlanner_CreateTable(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{}, columns); diff != "" { + if diff := cmp.Diff([]*pilosa.WireQueryField{}, columns); diff != "" { t.Fatal(diff) } }) @@ -523,21 +523,21 @@ func TestPlanner_CreateTable(t *testing.T) { if err != nil { t.Fatal(err) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "name", Type: parser.NewDataTypeString()}, - {ColumnName: "type", Type: parser.NewDataTypeString()}, - {ColumnName: "internal_type", Type: parser.NewDataTypeString()}, - {ColumnName: "created_at", Type: parser.NewDataTypeTimestamp()}, - {ColumnName: "keys", Type: parser.NewDataTypeBool()}, - {ColumnName: "cache_type", Type: parser.NewDataTypeString()}, - {ColumnName: "cache_size", Type: parser.NewDataTypeInt()}, - {ColumnName: "scale", Type: parser.NewDataTypeInt()}, - {ColumnName: "min", Type: parser.NewDataTypeInt()}, - {ColumnName: "max", Type: parser.NewDataTypeInt()}, - {ColumnName: "timeunit", Type: parser.NewDataTypeString()}, - {ColumnName: "epoch", Type: parser.NewDataTypeInt()}, - {ColumnName: "timequantum", Type: parser.NewDataTypeString()}, - {ColumnName: "ttl", Type: parser.NewDataTypeString()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldString("name"), + wireQueryFieldString("type"), + wireQueryFieldString("internal_type"), + wireQueryFieldTimestamp("created_at"), + wireQueryFieldBool("keys"), + wireQueryFieldString("cache_type"), + wireQueryFieldInt("cache_size"), + wireQueryFieldInt("scale"), + wireQueryFieldInt("min"), + wireQueryFieldInt("max"), + wireQueryFieldString("timeunit"), + wireQueryFieldInt("epoch"), + wireQueryFieldString("timequantum"), + wireQueryFieldString("ttl"), }, columns); diff != "" { t.Fatal(diff) } @@ -595,7 +595,7 @@ func TestPlanner_AlterTable(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{}, columns); diff != "" { + if diff := cmp.Diff([]*pilosa.WireQueryField{}, columns); diff != "" { t.Fatal(diff) } }) @@ -609,7 +609,7 @@ func TestPlanner_AlterTable(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{}, columns); diff != "" { + if diff := cmp.Diff([]*pilosa.WireQueryField{}, columns); diff != "" { t.Fatal(diff) } }) @@ -624,7 +624,7 @@ func TestPlanner_AlterTable(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{}, columns); diff != "" { + if diff := cmp.Diff([]*pilosa.WireQueryField{}, columns); diff != "" { t.Fatal(diff) } }) @@ -705,9 +705,9 @@ func TestPlanner_ExpressionsInSelectListParen(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "", Type: parser.NewDataTypeBool()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldBool(""), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -726,9 +726,9 @@ func TestPlanner_ExpressionsInSelectListParen(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "", Type: parser.NewDataTypeBool()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldBool(""), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -784,9 +784,9 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "", Type: parser.NewDataTypeBool()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldBool(""), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -805,9 +805,9 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "", Type: parser.NewDataTypeInt()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt(""), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -826,9 +826,9 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "", Type: parser.NewDataTypeInt()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt(""), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -851,9 +851,9 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "", Type: parser.NewDataTypeDecimal(2)}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldDecimal("", 2), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -872,9 +872,9 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "", Type: parser.NewDataTypeString()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldString(""), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -930,10 +930,10 @@ func TestPlanner_ExpressionsInSelectListCase(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "b", Type: parser.NewDataTypeInt()}, - {ColumnName: "", Type: parser.NewDataTypeInt()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("b"), + wireQueryFieldInt(""), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -952,10 +952,10 @@ func TestPlanner_ExpressionsInSelectListCase(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "b", Type: parser.NewDataTypeInt()}, - {ColumnName: "", Type: parser.NewDataTypeInt()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("b"), + wireQueryFieldInt(""), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -1013,10 +1013,10 @@ func TestPlanner_Select(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "a", Type: parser.NewDataTypeInt()}, - {ColumnName: "b", Type: parser.NewDataTypeInt()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("a"), + wireQueryFieldInt("b"), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -1035,10 +1035,10 @@ func TestPlanner_Select(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "a", Type: parser.NewDataTypeInt()}, - {ColumnName: "b", Type: parser.NewDataTypeInt()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("a"), + wireQueryFieldInt("b"), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -1057,10 +1057,10 @@ func TestPlanner_Select(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "foo", Type: parser.NewDataTypeInt()}, - {ColumnName: "bar", Type: parser.NewDataTypeInt()}, - {ColumnName: "baz", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("foo"), + wireQueryFieldInt("bar"), + wireQueryFieldID("baz"), }, columns); diff != "" { t.Fatal(diff) } @@ -1079,10 +1079,10 @@ func TestPlanner_Select(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "_id", Type: parser.NewDataTypeID()}, - {ColumnName: "a", Type: parser.NewDataTypeInt()}, - {ColumnName: "b", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldID("_id"), + wireQueryFieldInt("a"), + wireQueryFieldInt("b"), }, columns); diff != "" { t.Fatal(diff) } @@ -1101,10 +1101,10 @@ func TestPlanner_Select(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "_id", Type: parser.NewDataTypeID()}, - {ColumnName: "a", Type: parser.NewDataTypeInt()}, - {ColumnName: "b", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldID("_id"), + wireQueryFieldInt("a"), + wireQueryFieldInt("b"), }, columns); diff != "" { t.Fatal(diff) } @@ -1123,10 +1123,10 @@ func TestPlanner_Select(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "_id", Type: parser.NewDataTypeID()}, - {ColumnName: "a", Type: parser.NewDataTypeInt()}, - {ColumnName: "b", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldID("_id"), + wireQueryFieldInt("a"), + wireQueryFieldInt("b"), }, columns); diff != "" { t.Fatal(diff) } @@ -1145,9 +1145,9 @@ func TestPlanner_Select(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "a", Type: parser.NewDataTypeInt()}, - {ColumnName: "b", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("a"), + wireQueryFieldInt("b"), }, columns); diff != "" { t.Fatal(diff) } @@ -1201,10 +1201,10 @@ func TestPlanner_SelectOrderBy(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "a", Type: parser.NewDataTypeInt()}, - {ColumnName: "b", Type: parser.NewDataTypeInt()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("a"), + wireQueryFieldInt("b"), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -1471,8 +1471,8 @@ func TestPlanner_BulkInsert(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt(""), }, columns); diff != "" { t.Fatal(diff) } @@ -1628,10 +1628,10 @@ func TestPlanner_SelectSelectSource(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "a", Type: parser.NewDataTypeInt()}, - {ColumnName: "b", Type: parser.NewDataTypeInt()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("a"), + wireQueryFieldInt("b"), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -1650,10 +1650,10 @@ func TestPlanner_SelectSelectSource(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "a", Type: parser.NewDataTypeInt()}, - {ColumnName: "b", Type: parser.NewDataTypeInt()}, - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("a"), + wireQueryFieldInt("b"), + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -1725,8 +1725,8 @@ func TestPlanner_In(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "count", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("count"), }, columns); diff != "" { t.Fatal(diff) } @@ -1745,8 +1745,8 @@ func TestPlanner_In(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {Name: "count", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("count"), }, columns); diff != "" { t.Fatal(diff) } @@ -1764,8 +1764,8 @@ func TestPlanner_In(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {Name: "count", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("count"), }, columns); diff != "" { t.Fatal(diff) } @@ -1783,8 +1783,8 @@ func TestPlanner_In(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {Name: "count", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("count"), }, columns); diff != "" { t.Fatal(diff) } @@ -1855,8 +1855,8 @@ func TestPlanner_Distinct(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "_id", Type: parser.NewDataTypeID()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldID("_id"), }, columns); diff != "" { t.Fatal(diff) } @@ -1877,8 +1877,8 @@ func TestPlanner_Distinct(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "parentid", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("parentid"), }, columns); diff != "" { t.Fatal(diff) } @@ -1898,9 +1898,9 @@ func TestPlanner_Distinct(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "_id", Type: parser.NewDataTypeID()}, - {ColumnName: "parentid", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldID("_id"), + wireQueryFieldInt("parentid"), }, columns); diff != "" { t.Fatal(diff) } @@ -1950,10 +1950,10 @@ func TestPlanner_SelectTop(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "_id", Type: parser.NewDataTypeID()}, - {ColumnName: "a", Type: parser.NewDataTypeInt()}, - {ColumnName: "b", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldID("_id"), + wireQueryFieldInt("a"), + wireQueryFieldInt("b"), }, columns); diff != "" { t.Fatal(diff) } @@ -1973,12 +1973,60 @@ func TestPlanner_SelectTop(t *testing.T) { t.Fatal(diff) } - if diff := cmp.Diff([]*planner_types.PlannerColumn{ - {ColumnName: "_id", Type: parser.NewDataTypeID()}, - {ColumnName: "a", Type: parser.NewDataTypeInt()}, - {ColumnName: "b", Type: parser.NewDataTypeInt()}, + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldID("_id"), + wireQueryFieldInt("a"), + wireQueryFieldInt("b"), }, columns); diff != "" { t.Fatal(diff) } }) } + +// helpers + +func wireQueryFieldID(name string) *pilosa.WireQueryField { + return &pilosa.WireQueryField{ + Name: dax.FieldName(name), + Type: dax.BaseTypeID, + BaseType: dax.BaseTypeID, + } +} +func wireQueryFieldBool(name string) *pilosa.WireQueryField { + return &pilosa.WireQueryField{ + Name: dax.FieldName(name), + Type: dax.BaseTypeBool, + BaseType: dax.BaseTypeBool, + } +} +func wireQueryFieldString(name string) *pilosa.WireQueryField { + return &pilosa.WireQueryField{ + Name: dax.FieldName(name), + Type: dax.BaseTypeString, + BaseType: dax.BaseTypeString, + } +} +func wireQueryFieldInt(name string) *pilosa.WireQueryField { + return &pilosa.WireQueryField{ + Name: dax.FieldName(name), + Type: dax.BaseTypeInt, + BaseType: dax.BaseTypeInt, + } +} +func wireQueryFieldTimestamp(name string) *pilosa.WireQueryField { + return &pilosa.WireQueryField{ + Name: dax.FieldName(name), + Type: dax.BaseTypeTimestamp, + BaseType: dax.BaseTypeTimestamp, + } +} +func wireQueryFieldDecimal(name string, scale int64) *pilosa.WireQueryField { + return &pilosa.WireQueryField{ + Name: dax.FieldName(name), + Type: fmt.Sprintf("%s(%d)", dax.BaseTypeDecimal, scale), + BaseType: dax.BaseTypeDecimal, + TypeInfo: map[string]interface{}{ + "scale": scale, + }, + } +} diff --git a/sql3/sql_test.go b/sql3/sql_test.go index f21a2f0c6..be976f941 100644 --- a/sql3/sql_test.go +++ b/sql3/sql_test.go @@ -7,6 +7,7 @@ import ( "sort" "testing" + "github.com/molecula/featurebase/v3/dax" sql_test "github.com/molecula/featurebase/v3/sql3/test" "github.com/molecula/featurebase/v3/sql3/test/defs" "github.com/molecula/featurebase/v3/test" @@ -56,9 +57,9 @@ func TestSQL_Execute(t *testing.T) { assert.ElementsMatch(t, sqltest.ExpHdrs, headers) // make a map of column name to header index - m := make(map[string]int) + m := make(map[dax.FieldName]int) for i := range headers { - m[headers[i].ColumnName] = i + m[headers[i].Name] = i } // Put the expRows in the same column order as the headers returned @@ -67,7 +68,7 @@ func TestSQL_Execute(t *testing.T) { for i := range sqltest.ExpRows { exp[i] = make([]interface{}, len(headers)) for j := range sqltest.ExpHdrs { - targetIdx := m[sqltest.ExpHdrs[j].ColumnName] + targetIdx := m[sqltest.ExpHdrs[j].Name] assert.GreaterOrEqual(t, len(sqltest.ExpRows[i]), len(headers), "expected row set has fewer columns than returned headers") exp[i][targetIdx] = sqltest.ExpRows[i][j] diff --git a/sql3/test/defs/defs_aggregate.go b/sql3/test/defs/defs_aggregate.go index 757b1401a..95e556a26 100644 --- a/sql3/test/defs/defs_aggregate.go +++ b/sql3/test/defs/defs_aggregate.go @@ -1,8 +1,9 @@ package defs import ( + featurebase "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/dax" "github.com/molecula/featurebase/v3/pql" - "github.com/molecula/featurebase/v3/sql3/parser" ) // aggregate function tests @@ -328,7 +329,11 @@ var avgTests = TableTest{ "SELECT avg(d1) AS avg_rows FROM avg_test", ), ExpHdrs: hdrs( - hdr("avg_rows", parser.NewDataTypeDecimal(4)), + hdr("avg_rows", featurebase.WireQueryField{ + Type: dax.BaseTypeDecimal + "(4)", + BaseType: dax.BaseTypeDecimal, + TypeInfo: map[string]interface{}{"scale": int64(4)}, + }), ), ExpRows: rows( row(pql.NewDecimal(113333, 4)), diff --git a/sql3/test/defs/defs_between.go b/sql3/test/defs/defs_between.go index 059a68de7..bf6a87e7b 100644 --- a/sql3/test/defs/defs_between.go +++ b/sql3/test/defs/defs_between.go @@ -48,13 +48,13 @@ var betweenTests = TableTest{ SQLs: sqls( "select b1 between true and false from between_all_types", ), - ExpErr: "type 'BOOL' cannot be used a range subscript", + ExpErr: "type 'BOOL' cannot be used as a range subscript", }, { SQLs: sqls( "select d1 between 1.23 and 4.56 from between_all_types", ), - ExpErr: "type 'DECIMAL(2)' cannot be used a range subscript", + ExpErr: "type 'DECIMAL(2)' cannot be used as a range subscript", }, { SQLs: sqls( @@ -72,19 +72,19 @@ var betweenTests = TableTest{ SQLs: sqls( "select ids1 between [100, 102] and [456, 789] from between_all_types", ), - ExpErr: "type 'IDSET' cannot be used a range subscript", + ExpErr: "type 'IDSET' cannot be used as a range subscript", }, { SQLs: sqls( "select s1 between 'foo' and 'bar' from between_all_types", ), - ExpErr: "type 'STRING' cannot be used a range subscript", + ExpErr: "type 'STRING' cannot be used as a range subscript", }, { SQLs: sqls( "select ss1 between ['a', 'b'] and ['c', 'd'] from between_all_types", ), - ExpErr: "type 'STRINGSET' cannot be used a range subscript", + ExpErr: "type 'STRINGSET' cannot be used as a range subscript", }, { SQLs: sqls( @@ -149,13 +149,13 @@ var notBetweenTests = TableTest{ SQLs: sqls( "select b1 not between true and false from not_between_all_types", ), - ExpErr: "type 'BOOL' cannot be used a range subscript", + ExpErr: "type 'BOOL' cannot be used as a range subscript", }, { SQLs: sqls( "select d1 not between 1.23 and 4.56 from not_between_all_types", ), - ExpErr: "type 'DECIMAL(2)' cannot be used a range subscript", + ExpErr: "type 'DECIMAL(2)' cannot be used as a range subscript", }, { SQLs: sqls( @@ -173,19 +173,19 @@ var notBetweenTests = TableTest{ SQLs: sqls( "select ids1 not between [100, 102] and [456, 789] from not_between_all_types", ), - ExpErr: "type 'IDSET' cannot be used a range subscript", + ExpErr: "type 'IDSET' cannot be used as a range subscript", }, { SQLs: sqls( "select s1 not between 'foo' and 'bar' from not_between_all_types", ), - ExpErr: "type 'STRING' cannot be used a range subscript", + ExpErr: "type 'STRING' cannot be used as a range subscript", }, { SQLs: sqls( "select ss1 not between ['a', 'b'] and ['c', 'd'] from not_between_all_types", ), - ExpErr: "type 'STRINGSET' cannot be used a range subscript", + ExpErr: "type 'STRINGSET' cannot be used as a range subscript", }, { SQLs: sqls( diff --git a/sql3/test/defs/defs_groupby.go b/sql3/test/defs/defs_groupby.go index 399dbb0ce..e7e4dd8f6 100644 --- a/sql3/test/defs/defs_groupby.go +++ b/sql3/test/defs/defs_groupby.go @@ -1,8 +1,9 @@ package defs import ( + featurebase "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/dax" "github.com/molecula/featurebase/v3/pql" - "github.com/molecula/featurebase/v3/sql3/parser" ) // groupby tests @@ -109,7 +110,11 @@ var groupByTests = TableTest{ "select avg(i1) as avg_rows, i1 from groupby_test group by i1", ), ExpHdrs: hdrs( - hdr("avg_rows", parser.NewDataTypeDecimal(4)), + hdr("avg_rows", featurebase.WireQueryField{ + Type: dax.BaseTypeDecimal + "(4)", + BaseType: dax.BaseTypeDecimal, + TypeInfo: map[string]interface{}{"scale": int64(4)}, + }), hdr("i1", fldTypeInt), ), ExpRows: rows( @@ -125,7 +130,11 @@ var groupByTests = TableTest{ "select avg(d1) as avg_rows, i1 from groupby_test group by i1", ), ExpHdrs: hdrs( - hdr("avg_rows", parser.NewDataTypeDecimal(4)), + hdr("avg_rows", featurebase.WireQueryField{ + Type: dax.BaseTypeDecimal + "(4)", + BaseType: dax.BaseTypeDecimal, + TypeInfo: map[string]interface{}{"scale": int64(4)}, + }), hdr("i1", fldTypeInt), ), ExpRows: rows( diff --git a/sql3/test/defs/types.go b/sql3/test/defs/types.go index 700102a36..c21ec31c9 100644 --- a/sql3/test/defs/types.go +++ b/sql3/test/defs/types.go @@ -8,23 +8,46 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v3/sql3/parser" - planner_types "github.com/molecula/featurebase/v3/sql3/planner/types" + featurebase "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/dax" ) -type fldType parser.ExprDataType - // fldType constants are providing a map of a defined test type to the -// parser.ExprDataType +// featurebase.WireQueryField. var ( - fldTypeID fldType = parser.NewDataTypeID() - fldTypeBool fldType = parser.NewDataTypeBool() - fldTypeIDSet fldType = parser.NewDataTypeIDSet() - fldTypeInt fldType = parser.NewDataTypeInt() - fldTypeDecimal2 fldType = parser.NewDataTypeDecimal(2) - fldTypeString fldType = parser.NewDataTypeString() - fldTypeStringSet fldType = parser.NewDataTypeStringSet() - fldTypeTimestamp fldType = parser.NewDataTypeTimestamp() + fldTypeID featurebase.WireQueryField = featurebase.WireQueryField{ + Type: dax.BaseTypeID, + BaseType: dax.BaseTypeID, + } + fldTypeBool featurebase.WireQueryField = featurebase.WireQueryField{ + Type: dax.BaseTypeBool, + BaseType: dax.BaseTypeBool, + } + fldTypeIDSet featurebase.WireQueryField = featurebase.WireQueryField{ + Type: dax.BaseTypeIDSet, + BaseType: dax.BaseTypeIDSet, + } + fldTypeInt featurebase.WireQueryField = featurebase.WireQueryField{ + Type: dax.BaseTypeInt, + BaseType: dax.BaseTypeInt, + } + fldTypeDecimal2 featurebase.WireQueryField = featurebase.WireQueryField{ + Type: dax.BaseTypeDecimal + "(2)", + BaseType: dax.BaseTypeDecimal, + TypeInfo: map[string]interface{}{"scale": int64(2)}, + } + fldTypeString featurebase.WireQueryField = featurebase.WireQueryField{ + Type: dax.BaseTypeString, + BaseType: dax.BaseTypeString, + } + fldTypeStringSet featurebase.WireQueryField = featurebase.WireQueryField{ + Type: dax.BaseTypeStringSet, + BaseType: dax.BaseTypeStringSet, + } + fldTypeTimestamp featurebase.WireQueryField = featurebase.WireQueryField{ + Type: dax.BaseTypeTimestamp, + BaseType: dax.BaseTypeTimestamp, + } ) type compareMethod string @@ -84,7 +107,7 @@ func (tt TableTest) InsertInto(t *testing.T) string { type SQLTest struct { name string SQLs []string - ExpHdrs []*planner_types.PlannerColumn + ExpHdrs []*featurebase.WireQueryField ExpRows [][]interface{} ExpErr string Compare compareMethod @@ -107,7 +130,7 @@ type PQLTest struct { name string PQLs []string Table string - ExpHdrs []*planner_types.PlannerColumn + ExpHdrs []*featurebase.WireQueryField ExpRows [][]interface{} ExpErr string } @@ -126,7 +149,7 @@ func (s PQLTest) Name(i int) string { // The following "source" types are helpers for creating a test table. type sourceColumn struct { name string - typ fldType + typ featurebase.WireQueryField options string } @@ -142,7 +165,7 @@ func srcHdrs(hdrs ...sourceColumn) []sourceColumn { return hdrs } -func srcHdr(name string, typ fldType, opts ...string) sourceColumn { +func srcHdr(name string, typ featurebase.WireQueryField, opts ...string) sourceColumn { return sourceColumn{ name: name, typ: typ, @@ -227,7 +250,7 @@ func (s source) createTable() string { cols := []string{} for _, col := range s.columns { - f := col.name + " " + col.typ.TypeName() + f := col.name + " " + col.typ.Type if col.options != "" { f += " " + col.options } @@ -248,15 +271,18 @@ func (s source) insertInto(t *testing.T) string { } // hdrs is just a helper function to make the test definition look cleaner. -func hdrs(hdrs ...*planner_types.PlannerColumn) []*planner_types.PlannerColumn { +func hdrs(hdrs ...*featurebase.WireQueryField) []*featurebase.WireQueryField { return hdrs } -// hdr is just a helper function to make the test definition look cleaner. -func hdr(name string, typ fldType) *planner_types.PlannerColumn { - return &planner_types.PlannerColumn{ - ColumnName: name, - Type: typ, +// hdr is just a helper function to make the test definition look cleaner. It +// applies the `name` value to the provided WireQueryType. +func hdr(name string, typ featurebase.WireQueryField) *featurebase.WireQueryField { + return &featurebase.WireQueryField{ + Name: dax.FieldName(name), + Type: typ.Type, + BaseType: typ.BaseType, + TypeInfo: typ.TypeInfo, } } diff --git a/sql3/test/helpers.go b/sql3/test/helpers.go index 77066d055..a49a12411 100644 --- a/sql3/test/helpers.go +++ b/sql3/test/helpers.go @@ -3,14 +3,16 @@ package test import ( "context" + "strings" "testing" - pilosa "github.com/molecula/featurebase/v3" - planner_types "github.com/molecula/featurebase/v3/sql3/planner/types" + featurebase "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/dax" + plannertypes "github.com/molecula/featurebase/v3/sql3/planner/types" ) // MustQueryRows returns the row results as a slice of []interface{}, along with the columns. -func MustQueryRows(tb testing.TB, svr *pilosa.Server, q string) ([][]interface{}, []*planner_types.PlannerColumn, error) { +func MustQueryRows(tb testing.TB, svr *featurebase.Server, q string) ([][]interface{}, []*featurebase.WireQueryField, error) { tb.Helper() ctx := context.Background() @@ -32,26 +34,28 @@ func MustQueryRows(tb testing.TB, svr *pilosa.Server, q string) ([][]interface{} results := make([][]interface{}, 0) next, err := rowIter.Next(ctx) - if err != nil && err != planner_types.ErrNoMoreRows { + if err != nil && err != plannertypes.ErrNoMoreRows { return nil, nil, err } - for err != planner_types.ErrNoMoreRows { + for err != plannertypes.ErrNoMoreRows { result := make([]interface{}, len(ocolumns)) for i := range result { result[i] = next[i] } results = append(results, result) next, err = rowIter.Next(ctx) - if err != nil && err != planner_types.ErrNoMoreRows { + if err != nil && err != plannertypes.ErrNoMoreRows { return nil, nil, err } } //temporarily transform to Columns() - cols := make([]*planner_types.PlannerColumn, 0) + cols := make([]*featurebase.WireQueryField, 0) for _, oc := range ocolumns { - cols = append(cols, &planner_types.PlannerColumn{ - ColumnName: oc.ColumnName, - Type: oc.Type, + cols = append(cols, &featurebase.WireQueryField{ + Name: dax.FieldName(oc.ColumnName), + Type: strings.ToLower(oc.Type.TypeDescription()), + BaseType: dax.BaseType(strings.ToLower(oc.Type.TypeName())), + TypeInfo: oc.Type.TypeInfo(), }) } return results, cols, nil diff --git a/sql_test.go b/sql_test.go new file mode 100644 index 000000000..a47587d98 --- /dev/null +++ b/sql_test.go @@ -0,0 +1,95 @@ +package pilosa_test + +import ( + "encoding/json" + "testing" + + featurebase "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/pql" + "github.com/stretchr/testify/assert" +) + +func TestSQL(t *testing.T) { + t.Run("SQLResponse", func(t *testing.T) { + t.Run("Error", func(t *testing.T) { + body := `{"error": "bad"}` + sqlResponse := &featurebase.WireQueryResponse{} + err := json.Unmarshal([]byte(body), sqlResponse) + assert.NoError(t, err) + assert.Equal(t, "bad", sqlResponse.Error) + }) + + t.Run("Typed", func(t *testing.T) { + body := `{ + "schema":{ + "fields":[ + {"name":"an_int","type":"int","base-type":"int"}, + {"name":"a_decimal","type":"decimal","base-type":"decimal","type-info":{"scale":2}}, + {"name":"a_stringset","type":"stringset","base-type":"stringset"}, + {"name":"an_idset","type":"idset","base-type":"idset"}, + {"name":"a_bool","type":"bool","base-type":"bool"}, + {"name":"a_string","type":"string","base-type":"string"}, + {"name":"an_id","type":"id","base-type":"id"} + ] + }, + "data":[ + [1, 12.34, ["foo", "bar"], [4,5], true, "foobar", 8] + ], + "error":"", + "warnings":null + }` + sqlResponse := &featurebase.WireQueryResponse{} + //err := json.Unmarshal([]byte(body), sqlResponse) + err := sqlResponse.UnmarshalJSONTyped([]byte(body), true) + assert.NoError(t, err) + + row0 := sqlResponse.Data[0] + + assert.Equal(t, int64(1), row0[0]) + assert.Equal(t, pql.NewDecimal(1234, 2), row0[1]) + assert.Equal(t, featurebase.StringSet([]string{"foo", "bar"}), row0[2]) + assert.Equal(t, featurebase.IDSet([]int64{4, 5}), row0[3]) + assert.Equal(t, true, row0[4]) + assert.Equal(t, "foobar", row0[5]) + assert.Equal(t, int64(8), row0[6]) + + // Check the set stringers. + assert.Equal(t, "['foo', 'bar']", row0[2].(featurebase.StringSet).String()) + assert.Equal(t, "[4, 5]", row0[3].(featurebase.IDSet).String()) + }) + + t.Run("Untyped", func(t *testing.T) { + body := `{ + "schema":{ + "fields":[ + {"name":"an_int","type":"int","base-type":"int"}, + {"name":"a_decimal","type":"decimal","base-type":"decimal","type-info":{"scale":2}}, + {"name":"a_stringset","type":"stringset","base-type":"stringset"}, + {"name":"an_idset","type":"idset","base-type":"idset"}, + {"name":"a_bool","type":"bool","base-type":"bool"}, + {"name":"a_string","type":"string","base-type":"string"}, + {"name":"an_id","type":"id","base-type":"id"} + ] + }, + "data":[ + [1, 12.34, ["foo", "bar"], [4,5], true, "foobar", 8] + ], + "error":"", + "warnings":null + }` + sqlResponse := &featurebase.WireQueryResponse{} + err := json.Unmarshal([]byte(body), sqlResponse) + assert.NoError(t, err) + + row0 := sqlResponse.Data[0] + + assert.Equal(t, int64(1), row0[0]) + assert.Equal(t, pql.NewDecimal(1234, 2), row0[1]) + assert.Equal(t, []string{"foo", "bar"}, row0[2]) + assert.Equal(t, []int64{4, 5}, row0[3]) + assert.Equal(t, true, row0[4]) + assert.Equal(t, "foobar", row0[5]) + assert.Equal(t, int64(8), row0[6]) + }) + }) +} diff --git a/wire_response.go b/wire_response.go new file mode 100644 index 000000000..c7464ff13 --- /dev/null +++ b/wire_response.go @@ -0,0 +1,191 @@ +package pilosa + +import ( + "encoding/json" + "fmt" + "log" + "strings" + + "github.com/molecula/featurebase/v3/dax" + "github.com/molecula/featurebase/v3/pql" + "github.com/pkg/errors" +) + +// WireQueryResponse is the standard featurebase response type which can be +// serialized and sent over the wire. +type WireQueryResponse struct { + Schema WireQuerySchema `json:"schema"` + Data [][]interface{} `json:"data"` + Error string `json:"error"` + Warnings []string `json:"warnings"` + QueryPlan map[string]interface{} `json:"query-plan"` + ExecutionTime int64 `json:"execution-time"` +} + +// WireQuerySchema is a list of Fields which map to the data columns in the +// Response. +type WireQuerySchema struct { + Fields []*WireQueryField `json:"fields"` +} + +// WireQueryField is a field name along with a supported BaseType and type +// information. +type WireQueryField struct { + Name dax.FieldName `json:"name"` + Type string `json:"type"` // human readable display (e.g. "decimal(2)") + BaseType dax.BaseType `json:"base-type"` // for programmatic switching on type (e.g. "decimal") + TypeInfo map[string]interface{} `json:"type-info"` // type modifiers (like scale), but not constraints (like min/max) +} + +// UnmarshalJSON is a custom unmarshaller for the SQLResponse that converts the +// value types in `Data` based on the types in `Schema`. +func (s *WireQueryResponse) UnmarshalJSON(in []byte) error { + return s.UnmarshalJSONTyped(in, false) +} + +// UnmarshalJSONTyped is a temporary until we send typed values back in sql +// responses. At that point, we can get rid of the typed=false path. In order to +// do that, we need sql3 to return typed values, and we need the sql3/test/defs +// to define results as typed values (like `IDSet`) instead of (for example) +// `[]int64`. +func (s *WireQueryResponse) UnmarshalJSONTyped(in []byte, typed bool) error { + type Alias WireQueryResponse + var aux Alias + + if err := json.Unmarshal(in, &aux); err != nil { + return err + } + + *s = WireQueryResponse(aux) + + // If the SQLResponse contains an error, don't bother doing any conversions + // on the data. + if s.Error != "" { + return nil + } + + // Try to convert the types in the TypeInfo map for each field in the + // schema. + for _, fld := range s.Schema.Fields { + // TODO(tlt): we can remove these two "ToLower" calls once sql3 is + // returning dax.FieldType (i.e. lowercase). + fld.Type = strings.ToLower(fld.Type) + fld.BaseType = dax.BaseType(strings.ToLower(string(fld.BaseType))) + for k, v := range fld.TypeInfo { + switch k { + case "scale": + fld.TypeInfo[k] = int64(v.(float64)) + } + } + } + + // Try to convert the data types based on the headers. + for i := range s.Data { + for j, hdr := range s.Schema.Fields { + switch hdr.BaseType { + case dax.BaseTypeID, dax.BaseTypeInt: + if _, ok := s.Data[i][j].(float64); ok { + s.Data[i][j] = int64(s.Data[i][j].(float64)) + } + + case dax.BaseTypeIDSet: + if src, ok := s.Data[i][j].([]interface{}); ok { + if typed { + val := make(IDSet, len(src)) + for k := range src { + val[k] = int64(src[k].(float64)) + } + s.Data[i][j] = val + } else { + val := make([]int64, len(src)) + for k := range src { + val[k] = int64(src[k].(float64)) + } + s.Data[i][j] = val + } + } + + case dax.BaseTypeDecimal: + if _, ok := s.Data[i][j].(float64); ok { + var scale int64 + if scaleVal, ok := hdr.TypeInfo["scale"]; !ok { + return errors.New("decimal does not have a scale") + } else if scaleInt64, ok := scaleVal.(int64); !ok { + return errors.New("scale can't be cast to int64") + } else { + scale = scaleInt64 + } + + format := fmt.Sprintf("%%.%df", scale) + dec, err := pql.ParseDecimal(fmt.Sprintf(format, s.Data[i][j])) + if err != nil { + return errors.Wrap(err, "parsing decimal") + } + if dec.Scale != scale { + dec = pql.NewDecimal(dec.ToInt64(scale), scale) + } + s.Data[i][j] = dec + } + + case dax.BaseTypeStringSet: + if src, ok := s.Data[i][j].([]interface{}); ok { + if typed { + val := make(StringSet, len(src)) + for k := range src { + val[k] = src[k].(string) + } + s.Data[i][j] = val + } else { + val := make([]string, len(src)) + for k := range src { + val[k] = src[k].(string) + } + s.Data[i][j] = val + } + } + + case dax.BaseTypeBool, dax.BaseTypeString: + // no need to convert + + default: + log.Printf("WARNING: unimplemented: %T", hdr.BaseType) + } + } + } + + return nil +} + +// IDSet is a return type specific to SQLResponse types. +type IDSet []int64 + +func (ii IDSet) String() string { + var sb strings.Builder + sb.WriteString("[") + for i := range ii { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(fmt.Sprintf("%d", ii[i])) + } + sb.WriteString("]") + + return sb.String() +} + +// StringSet is a return type specific to SQLResponse types. +type StringSet []string + +func (ss StringSet) String() string { + var sb strings.Builder + sb.WriteString("[") + for i := range ss { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString("'" + ss[i] + "'") + } + sb.WriteString("]") + + return sb.String() +}