From 9386fc75b259e4143146cd0a82dfbacb1b490e90 Mon Sep 17 00:00:00 2001 From: Pat Okeeffe <85502298+paddyjok@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:06:58 -0600 Subject: [PATCH] implement select from time quantum columns (fb-1654) (#2282) * first time quantum queries working * implement select from timequantum columns * skip a dax test * make linter happy * addressed review feedback * reverted over eager test elimination --- dax/table.go | 22 ++-- dax/test/dax/dax_test.go | 1 + schema.go | 48 ++++---- sql3/errors.go | 38 +++++- sql3/parser/astdatatype.go | 7 +- sql3/planner/compilecreatetable.go | 35 +++--- sql3/planner/compileselect.go | 30 +---- sql3/planner/expression.go | 16 +++ sql3/planner/expressionanalyzercall.go | 6 +- sql3/planner/expressionpql.go | 79 +++++++++++++ sql3/planner/expressiontypes.go | 4 + sql3/planner/inbuiltfunctionsquantum.go | 46 ++++++++ sql3/planner/inbuiltfunctionsset.go | 4 +- sql3/planner/inbuiltfunctionstable.go | 67 ----------- sql3/planner/oppqldistinctscan.go | 8 ++ sql3/planner/oppqltablescan.go | 94 +++++++++++---- sql3/planner/oprelalias.go | 34 ++++++ sql3/planner/opsystemtable.go | 2 +- sql3/planner/optablevaluedfunction.go | 5 +- sql3/planner/planoptimizer.go | 150 +++++++++++------------- sql3/planner/types/operator.go | 2 + sql3/sql_complex_test.go | 18 +-- sql3/test/defs/defs.go | 3 +- sql3/test/defs/defs_join.go | 4 +- sql3/test/defs/defs_timequantum.go | 83 +++++++++++-- sql3/test/defs/types.go | 8 ++ 26 files changed, 531 insertions(+), 283 deletions(-) create mode 100644 sql3/planner/inbuiltfunctionsquantum.go delete mode 100644 sql3/planner/inbuiltfunctionstable.go diff --git a/dax/table.go b/dax/table.go index c77ffdbed..37bcc0179 100644 --- a/dax/table.go +++ b/dax/table.go @@ -73,14 +73,16 @@ const PrefixTable = "tbl" // Base types. const ( - BaseTypeBool = "bool" // - BaseTypeDecimal = "decimal" // - BaseTypeID = "id" // non-keyed mutex - BaseTypeIDSet = "idset" // non-keyed set - BaseTypeInt = "int" // - BaseTypeString = "string" // keyed mutex - BaseTypeStringSet = "stringset" // keyed set - BaseTypeTimestamp = "timestamp" // + BaseTypeBool = "bool" // + BaseTypeDecimal = "decimal" // + BaseTypeID = "id" // non-keyed mutex + BaseTypeIDSet = "idset" // non-keyed set + BaseTypeIDSetQ = "idsetq" // non-keyed set timequantum + BaseTypeInt = "int" // + BaseTypeString = "string" // keyed mutex + BaseTypeStringSet = "stringset" // keyed set + BaseTypeStringSetQ = "stringsetq" // keyed set timequantum + BaseTypeTimestamp = "timestamp" // DefaultPartitionN = 256 @@ -679,9 +681,11 @@ func BaseTypeFromString(s string) (BaseType, error) { BaseTypeDecimal, BaseTypeID, BaseTypeIDSet, + BaseTypeIDSetQ, BaseTypeInt, BaseTypeString, BaseTypeStringSet, + BaseTypeStringSetQ, BaseTypeTimestamp: return BaseType(lowered), nil default: @@ -706,7 +710,7 @@ func (f *Field) String() string { // StringKeys returns true if the field uses string keys. func (f *Field) StringKeys() bool { switch f.Type { - case BaseTypeString, BaseTypeStringSet: + case BaseTypeString, BaseTypeStringSet, BaseTypeStringSetQ: return true } return false diff --git a/dax/test/dax/dax_test.go b/dax/test/dax/dax_test.go index a07a65918..ff0c3d5f4 100644 --- a/dax/test/dax/dax_test.go +++ b/dax/test/dax/dax_test.go @@ -143,6 +143,7 @@ func TestDAXIntegration(t *testing.T) { "viewtests/drop-view", // drop view does a delete "viewtests/drop-view-if-exists-after-drop", "viewtests/select-view-after-drop", + "time_quantum_insert/test-12", // orchestrator currently does not support to,from args on Rows() } doSkip := func(name string) bool { diff --git a/schema.go b/schema.go index 60d7897ba..e78a12f18 100644 --- a/schema.go +++ b/schema.go @@ -243,9 +243,9 @@ func FieldInfoToField(fi *FieldInfo) *dax.Field { fieldType = dax.BaseTypeBool case FieldTypeTime: if fo.Keys { - fieldType = dax.BaseTypeStringSet + fieldType = dax.BaseTypeStringSetQ } else { - fieldType = dax.BaseTypeIDSet + fieldType = dax.BaseTypeIDSetQ } timeQuantum = dax.TimeQuantum(fo.TimeQuantum) default: @@ -397,11 +397,13 @@ func fieldToFieldType(f *dax.Field) string { return string(f.Type) } return "mutex" + case dax.BaseTypeIDSet, dax.BaseTypeStringSet: - if f.Options.TimeQuantum != "" { - return "time" - } return "set" + + case dax.BaseTypeIDSetQ, dax.BaseTypeStringSetQ: + return "time" + default: return string(f.Type) } @@ -449,15 +451,13 @@ func FieldOptionsFromField(fld *dax.Field) ([]FieldOption, error) { OptFieldTypeMutex(cacheType, cacheSize), ) case dax.BaseTypeIDSet: - if fld.Options.TimeQuantum != "" { - opts = append(opts, - OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()), - ) - } else { - opts = append(opts, - OptFieldTypeSet(cacheType, cacheSize), - ) - } + opts = append(opts, + OptFieldTypeSet(cacheType, cacheSize), + ) + case dax.BaseTypeIDSetQ: + opts = append(opts, + OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()), + ) case dax.BaseTypeInt: opts = append(opts, OptFieldTypeInt(fld.Options.Min.ToInt64(0), fld.Options.Max.ToInt64(0)), @@ -468,17 +468,15 @@ func FieldOptionsFromField(fld *dax.Field) ([]FieldOption, error) { OptFieldKeys(), ) case dax.BaseTypeStringSet: - if fld.Options.TimeQuantum != "" { - opts = append(opts, - OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()), - OptFieldKeys(), - ) - } else { - opts = append(opts, - OptFieldTypeSet(cacheType, cacheSize), - OptFieldKeys(), - ) - } + opts = append(opts, + OptFieldTypeSet(cacheType, cacheSize), + OptFieldKeys(), + ) + case dax.BaseTypeStringSetQ: + opts = append(opts, + OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()), + OptFieldKeys(), + ) case dax.BaseTypeTimestamp: opts = append(opts, OptFieldTypeTimestamp(fld.Options.Epoch, fld.Options.TimeUnit), diff --git a/sql3/errors.go b/sql3/errors.go index 62354ddf8..fa3e69fe2 100644 --- a/sql3/errors.go +++ b/sql3/errors.go @@ -37,6 +37,7 @@ const ( ErrIntOrDecimalOrTimestampOrStringExpressionExpected errors.Code = "ErrIntOrDecimalOrTimestampOrStringExpressionExpected" ErrStringExpressionExpected errors.Code = "ErrStringExpressionExpected" ErrSetExpressionExpected errors.Code = "ErrSetExpressionExpected" + ErrTimeQuantumExpressionExpected errors.Code = "ErrTimeQuantumExpressionExpected" ErrSingleRowExpected errors.Code = "ErrSingleRowExpected" // type related errors @@ -58,7 +59,8 @@ const ( ErrInvalidColumnInFilterExpression errors.Code = "ErrInvalidColumnInFilterExpression" ErrInvalidTypeInFilterExpression errors.Code = "ErrInvalidTypeInFilterExpression" - ErrTypeAssignmentIncompatible errors.Code = "ErrTypeAssignmentIncompatible" + ErrTypeAssignmentIncompatible errors.Code = "ErrTypeAssignmentIncompatible" + ErrTypeAssignmentToTimeQuantumIncompatible errors.Code = "ErrTypeAssignmentToTimeQuantumIncompatible" ErrInvalidUngroupedColumnReference errors.Code = "ErrInvalidUngroupedColumnReference" ErrInvalidUngroupedColumnReferenceInHaving errors.Code = "ErrInvalidUngroupedColumnReferenceInHaving" @@ -132,6 +134,10 @@ const ( ErrValueOutOfRange errors.Code = "ErrValueOutOfRange" ErrStringLengthMismatch errors.Code = "ErrStringLengthMismatch" ErrUnexpectedTypeConversion errors.Code = "ErrUnexpectedTypeConversion" + + // time quantum function eval + ErrQRangeFromAndToTimeCannotBeBothNull errors.Code = "ErrQRangeFromAndToTimeCannotBeBothNull" + ErrQRangeInvalidUse errors.Code = "ErrQRangeInvalidUse" ) func NewErrDuplicateColumn(line int, col int, column string) error { @@ -235,6 +241,13 @@ func NewErrInvalidTypeCoercion(line, col int, from, to string) error { ) } +func NewErrTypeAssignmentToTimeQuantumIncompatible(line, col int, type1 string) error { + return errors.New( + ErrTypeAssignmentToTimeQuantumIncompatible, + fmt.Sprintf("[%d:%d] an expression of type '%s' cannot be assigned to a timequantum", line, col, type1), + ) +} + func NewErrLiteralExpected(line, col int) error { return errors.New( ErrLiteralExpected, @@ -445,6 +458,13 @@ func NewErrSetExpressionExpected(line, col int) error { ) } +func NewErrTimeQuantumExpressionExpected(line, col int) error { + return errors.New( + ErrTimeQuantumExpressionExpected, + fmt.Sprintf("[%d:%d] time quantum expression expected", line, col), + ) +} + func NewErrSingleRowExpected(line, col int) error { return errors.New( ErrSingleRowExpected, @@ -816,3 +836,19 @@ func NewErrUnexpectedTypeConversion(line, col int, val interface{}) error { NewErrInternalf("unexpected type conversion %T", val).Error(), ) } + +// time quantum function evaluation + +func NewErrQRangeFromAndToTimeCannotBeBothNull(line, col int) error { + return errors.New( + ErrQRangeFromAndToTimeCannotBeBothNull, + fmt.Sprintf("[%d:%d] calling ranqeq() 'from' and 'to' parameters cannot both be null", line, col), + ) +} + +func NewErrQRangeInvalidUse(line, col int) error { + return errors.New( + ErrQRangeInvalidUse, + fmt.Sprintf("[%d:%d] calling ranqeq() usage invalid", line, col), + ) +} diff --git a/sql3/parser/astdatatype.go b/sql3/parser/astdatatype.go index e2e604b1e..2b9114624 100644 --- a/sql3/parser/astdatatype.go +++ b/sql3/parser/astdatatype.go @@ -13,9 +13,11 @@ func IsValidTypeName(typeName string) bool { dax.BaseTypeDecimal, dax.BaseTypeID, dax.BaseTypeIDSet, + dax.BaseTypeIDSetQ, dax.BaseTypeInt, dax.BaseTypeString, dax.BaseTypeStringSet, + dax.BaseTypeStringSetQ, dax.BaseTypeTimestamp: return true default: @@ -238,7 +240,6 @@ func (*DataTypeIDSet) TypeInfo() map[string]interface{} { return nil } -// TODO (pok) should time quantum be it's own type and not a constraint? type DataTypeIDSetQuantum struct { } @@ -247,7 +248,7 @@ func NewDataTypeIDSetQuantum() *DataTypeIDSetQuantum { } func (*DataTypeIDSetQuantum) BaseTypeName() string { - return dax.BaseTypeIDSet + return dax.BaseTypeIDSetQ } func (dt *DataTypeIDSetQuantum) TypeDescription() string { @@ -323,7 +324,7 @@ func NewDataTypeStringSetQuantum() *DataTypeStringSetQuantum { } func (*DataTypeStringSetQuantum) BaseTypeName() string { - return dax.BaseTypeStringSet + return dax.BaseTypeStringSetQ } func (dt *DataTypeStringSetQuantum) TypeDescription() string { diff --git a/sql3/planner/compilecreatetable.go b/sql3/planner/compilecreatetable.go index 0712afc8e..f6d716418 100644 --- a/sql3/planner/compilecreatetable.go +++ b/sql3/planner/compilecreatetable.go @@ -196,19 +196,17 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column switch strings.ToLower(typeName) { case dax.BaseTypeBool: column.fos = append(column.fos, pilosa.OptFieldTypeBool()) - case dax.BaseTypeDecimal: + case dax.BaseTypeDecimal: // if we don't have a scale, it's an error if col.Type.Scale == nil { return nil, sql3.NewErrDecimalScaleExpected(col.Type.Name.NamePos.Line, col.Type.Name.NamePos.Column) } - // get the scale value scale, err = strconv.ParseInt(col.Type.Scale.Value, 10, 64) if err != nil { return nil, err } - // Adjust min/max to fit within the scaled min/max. scaledMin, scaledMax := pql.MinMax(scale) if scaledMax.LessThan(max) { @@ -217,30 +215,35 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column if scaledMin.GreaterThan(min) { min = scaledMin } - column.fos = append(column.fos, pilosa.OptFieldTypeDecimal(scale, min, max)) + case dax.BaseTypeID: column.fos = append(column.fos, pilosa.OptFieldTypeMutex(cacheType, cacheSize)) + case dax.BaseTypeIDSet: - if timeQuantum != "" { - column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl)) - } else { - column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize)) - } + column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize)) + + case dax.BaseTypeIDSetQ: + column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl)) + case dax.BaseTypeInt: column.fos = append(column.fos, pilosa.OptFieldTypeInt(min.ToInt64(0), max.ToInt64(0))) + case dax.BaseTypeString: column.fos = append(column.fos, pilosa.OptFieldTypeMutex(cacheType, cacheSize)) column.fos = append(column.fos, pilosa.OptFieldKeys()) + case dax.BaseTypeStringSet: - if timeQuantum != "" { - column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl)) - } else { - column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize)) - } + column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize)) column.fos = append(column.fos, pilosa.OptFieldKeys()) + + case dax.BaseTypeStringSetQ: + column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl)) + column.fos = append(column.fos, pilosa.OptFieldKeys()) + case dax.BaseTypeTimestamp: column.fos = append(column.fos, pilosa.OptFieldTypeTimestamp(epoch, timeUnit)) + } return column, nil } @@ -392,8 +395,8 @@ func (p *ExecutionPlanner) analyzeColumn(typeName string, col *parser.ColumnDefi handledConstraints[parser.TIMEUNIT] = struct{}{} case *parser.TimeQuantumConstraint: - //make sure we have a set type - if !(strings.EqualFold(typeName, dax.BaseTypeStringSet) || strings.EqualFold(typeName, dax.BaseTypeIDSet)) { + //make sure we have one of the time quantum types + if !(strings.EqualFold(typeName, dax.BaseTypeStringSetQ) || strings.EqualFold(typeName, dax.BaseTypeIDSetQ)) { return sql3.NewErrBadColumnConstraint(col.Name.NamePos.Line, col.Name.NamePos.Column, "TIMEQUANTUM", typeName) } //check the type of the expression diff --git a/sql3/planner/compileselect.go b/sql3/planner/compileselect.go index 940aff215..ddecdfd20 100644 --- a/sql3/planner/compileselect.go +++ b/sql3/planner/compileselect.go @@ -573,35 +573,7 @@ func (p *ExecutionPlanner) analyzeSource(ctx context.Context, source parser.Sour return source, nil case *parser.TableValuedFunction: - // check it actually is a table valued function - we only support one right now; subtable() - switch strings.ToUpper(source.Name.Name) { - case "SUBTABLE": - _, err := p.analyzeCallExpression(ctx, source.Call, scope) - if err != nil { - return nil, err - } - - tvfResultType, ok := source.Call.ResultDataType.(*parser.DataTypeSubtable) - if !ok { - return nil, sql3.NewErrInternalf("unexepected tvf return type") - } - - // populate the output columns from the source - for idx, member := range tvfResultType.Columns { - soc := &parser.SourceOutputColumn{ - TableName: "", // TODO (pok) use the tq column actually referenced as the table name - ColumnName: member.Name, - ColumnIndex: idx, - Datatype: member.DataType, - } - source.OutputColumns = append(source.OutputColumns, soc) - } - - default: - return nil, sql3.NewErrInternalf("table valued function expected") - } - - return source, nil + return nil, sql3.NewErrInternalf("table valued function expected") case *parser.SelectStatement: expr, err := p.analyzeSelectStatement(ctx, source) diff --git a/sql3/planner/expression.go b/sql3/planner/expression.go index 3e585910c..6a284a8d1 100644 --- a/sql3/planner/expression.go +++ b/sql3/planner/expression.go @@ -1576,6 +1576,9 @@ func (n *callPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er return n.EvaluateToTimestamp(currentRow) case "STR": return n.EvaluateStr(currentRow) + // time quantum functions + case "RANGEQ": + return n.EvaluateRangeQ(currentRow) default: return nil, sql3.NewErrInternalf("unhandled function name '%s'", n.name) } @@ -2063,6 +2066,19 @@ func (n *stringLiteralPlanExpression) WithChildren(children ...types.PlanExpress return n, nil } +func (expr *stringLiteralPlanExpression) ConvertToTimestamp() *time.Time { + //try to coerce to a date + if tm, err := time.ParseInLocation(time.RFC3339Nano, expr.value, time.UTC); err == nil { + return &tm + } else if tm, err := time.ParseInLocation(time.RFC3339, expr.value, time.UTC); err == nil { + return &tm + } else if tm, err := time.ParseInLocation("2006-01-02", expr.value, time.UTC); err == nil { + return &tm + } else { + return nil + } +} + // castPlanExpressionis a cast op type castPlanExpression struct { lhs types.PlanExpression diff --git a/sql3/planner/expressionanalyzercall.go b/sql3/planner/expressionanalyzercall.go index 698b994c8..51aaf78d1 100644 --- a/sql3/planner/expressionanalyzercall.go +++ b/sql3/planner/expressionanalyzercall.go @@ -216,8 +216,6 @@ func (p *ExecutionPlanner) analyzeCallExpression(ctx context.Context, call *pars case "DATEPART": return p.analyzeFunctionDatePart(call, scope) - case "SUBTABLE": - return p.analyzeFunctionSubtable(call, scope) case "REVERSE": return p.analyseFunctionReverse(call, scope) case "CHAR": @@ -258,6 +256,10 @@ func (p *ExecutionPlanner) analyzeCallExpression(ctx context.Context, call *pars return p.analyzeFunctionToTimestamp(call, scope) case "STR": return p.analyseFunctionStr(call, scope) + // time quantum funtions + case "RANGEQ": + return p.analyzeFunctionRangeQ(call, scope) + default: return nil, sql3.NewErrCallUnknownFunction(call.Name.NamePos.Line, call.Name.NamePos.Column, call.Name.Name) } diff --git a/sql3/planner/expressionpql.go b/sql3/planner/expressionpql.go index e0f7ea290..e2b1a6b1c 100644 --- a/sql3/planner/expressionpql.go +++ b/sql3/planner/expressionpql.go @@ -96,6 +96,85 @@ func (p *ExecutionPlanner) generatePQLCallFromExpr(ctx context.Context, expr typ } return call, nil + case "RANGEQ": + col := expr.args[0].(*qualifiedRefPlanExpression) + + var fromValue interface{} + switch fromExpr := expr.args[1].(type) { + case *stringLiteralPlanExpression: + // parse timestamp from string and use the int value + ts := fromExpr.ConvertToTimestamp() + if ts == nil { + return nil, sql3.NewErrInvalidTypeCoercion(0, 0, fromExpr.value, parser.NewDataTypeTimestamp().TypeDescription()) + } + fromValue = ts.Unix() + + case *intLiteralPlanExpression: + // use the int value + fromValue = fromExpr.value + + case *nullLiteralPlanExpression: + fromValue = nil + + default: + return nil, sql3.NewErrInternalf("unexpected argument type '%T'", expr.args[1]) + } + + var toValue interface{} + switch toExpr := expr.args[2].(type) { + case *stringLiteralPlanExpression: + // parse timestamp from string and use the int value + ts := toExpr.ConvertToTimestamp() + if ts == nil { + return nil, sql3.NewErrInvalidTypeCoercion(0, 0, toExpr.value, parser.NewDataTypeTimestamp().TypeDescription()) + } + toValue = ts.Unix() + + case *intLiteralPlanExpression: + // use the int value + toValue = toExpr.value + + case *nullLiteralPlanExpression: + toValue = nil + + default: + return nil, sql3.NewErrInternalf("unexpected argument type '%T'", expr.args[1]) + } + + if fromValue == nil && toValue == nil { + return nil, sql3.NewErrQRangeFromAndToTimeCannotBeBothNull(0, 0) + } + + var call *pql.Call + if fromValue == nil && toValue != nil { + call = &pql.Call{ + Name: "Rows", + Args: map[string]interface{}{ + "field": strings.ToLower(col.columnName), + "to": toValue, + }, + } + } else if fromValue != nil && toValue == nil { + call = &pql.Call{ + Name: "Rows", + Args: map[string]interface{}{ + "field": strings.ToLower(col.columnName), + "from": fromValue, + }, + } + } else { + call = &pql.Call{ + Name: "Rows", + Args: map[string]interface{}{ + "field": strings.ToLower(col.columnName), + "from": fromValue, + "to": toValue, + }, + } + } + + return call, nil + default: return nil, sql3.NewErrInternalf("unsupported scalar function '%s'", expr.name) } diff --git a/sql3/planner/expressiontypes.go b/sql3/planner/expressiontypes.go index 569dc1f0e..b7ee9b51e 100644 --- a/sql3/planner/expressiontypes.go +++ b/sql3/planner/expressiontypes.go @@ -456,6 +456,10 @@ func typeIsSet(testType parser.ExprDataType) (bool, parser.ExprDataType) { return true, parser.NewDataTypeID() case *parser.DataTypeStringSet: return true, parser.NewDataTypeString() + case *parser.DataTypeIDSetQuantum: + return true, parser.NewDataTypeID() + case *parser.DataTypeStringSetQuantum: + return true, parser.NewDataTypeString() default: return false, nil } diff --git a/sql3/planner/inbuiltfunctionsquantum.go b/sql3/planner/inbuiltfunctionsquantum.go new file mode 100644 index 000000000..21593ae48 --- /dev/null +++ b/sql3/planner/inbuiltfunctionsquantum.go @@ -0,0 +1,46 @@ +package planner + +import ( + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" +) + +func (p *ExecutionPlanner) analyzeFunctionRangeQ(call *parser.Call, scope parser.Statement) (parser.Expr, error) { + if len(call.Args) != 3 { + return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 3, len(call.Args)) + } + + // first arg should be time quantum type + ok, _ := typeIsTimeQuantum(call.Args[0].DataType()) + if !ok { + return nil, sql3.NewErrTimeQuantumExpressionExpected(call.Args[0].Pos().Line, call.Args[0].Pos().Column) + } + + // second is 'from' timestamp, can be null + targetType := parser.NewDataTypeTimestamp() + if !typesAreAssignmentCompatible(targetType, call.Args[1].DataType()) { + return nil, sql3.NewErrTypeAssignmentIncompatible(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Args[0].DataType().TypeDescription(), targetType.TypeDescription()) + } + + _, fromLiteralNull := call.Args[1].(*parser.NullLit) + + // second is 'to' timestamp, can be null + if !typesAreAssignmentCompatible(targetType, call.Args[2].DataType()) { + return nil, sql3.NewErrTypeAssignmentIncompatible(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Args[0].DataType().TypeDescription(), targetType.TypeDescription()) + } + _, toLiteralNull := call.Args[2].(*parser.NullLit) + + // if we have two literal nulls we have a problem + if fromLiteralNull && toLiteralNull { + return nil, sql3.NewErrQRangeFromAndToTimeCannotBeBothNull(call.Rparen.Line, call.Rparen.Column) + } + + call.ResultDataType = parser.NewDataTypeBool() + + return call, nil +} + +func (n *callPlanExpression) EvaluateRangeQ(currentRow []interface{}) (interface{}, error) { + // rangeq() should only ever be used as a push down filter for now - if we get to here, we should error + return nil, sql3.NewErrQRangeInvalidUse(0, 0) +} diff --git a/sql3/planner/inbuiltfunctionsset.go b/sql3/planner/inbuiltfunctionsset.go index 3c0b5a058..f696bb6c5 100644 --- a/sql3/planner/inbuiltfunctionsset.go +++ b/sql3/planner/inbuiltfunctionsset.go @@ -25,7 +25,7 @@ func (n *callPlanExpression) EvaluateSetContains(currentRow []interface{}) (inte if targetSetEval != nil { switch typ := n.args[0].Type().(type) { - case *parser.DataTypeStringSet: + case *parser.DataTypeStringSet, *parser.DataTypeStringSetQuantum: targetSet, ok := targetSetEval.([]string) if !ok { return nil, sql3.NewErrInternalf("unable to convert value") @@ -38,7 +38,7 @@ func (n *callPlanExpression) EvaluateSetContains(currentRow []interface{}) (inte return stringSetContains(targetSet, testValue), nil - case *parser.DataTypeIDSet: + case *parser.DataTypeIDSet, *parser.DataTypeIDSetQuantum: targetSet, ok := targetSetEval.([]int64) if !ok { return nil, sql3.NewErrInternalf("unable to convert value") diff --git a/sql3/planner/inbuiltfunctionstable.go b/sql3/planner/inbuiltfunctionstable.go deleted file mode 100644 index 19bc67c93..000000000 --- a/sql3/planner/inbuiltfunctionstable.go +++ /dev/null @@ -1,67 +0,0 @@ -package planner - -import ( - "github.com/featurebasedb/featurebase/v3/dax" - "github.com/featurebasedb/featurebase/v3/sql3" - "github.com/featurebasedb/featurebase/v3/sql3/parser" -) - -// TODO (pok) this needs to go somewhere -/*func (p *ExecutionPlanner) analyzeFunctionRecord(call *parser.Call, scope parser.Statement) (parser.Expr, error) { - if len(call.Args) != 2 { - return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 2, len(call.Args)) - } - // timestamp - timestampType := parser.NewDataTypeTimestamp() - if !typesAreAssignmentCompatible(timestampType, call.Args[0].DataType()) { - return nil, sql3.NewErrParameterTypeMistmatch(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Args[0].DataType().TypeName(), timestampType.TypeName()) - } - - // set - ok, _ := typeIsSet(call.Args[1].DataType()) - if !ok { - return nil, sql3.NewErrSetExpressionExpected(call.Args[1].Pos().Line, call.Args[1].Pos().Column) - } - - //return record - call.ResultDataType = parser.NewDataTypeSubtable([]*parser.SubtableColumn{ - { - Name: "", - DataType: timestampType, - }, - { - Name: "", - DataType: call.Args[1].DataType(), - }, - }) - - return call, nil -} -*/ - -func (p *ExecutionPlanner) analyzeFunctionSubtable(call *parser.Call, scope parser.Statement) (parser.Expr, error) { - if len(call.Args) != 1 { - return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 2, len(call.Args)) - } - // set - ok, _ := typeIsTimeQuantum(call.Args[0].DataType()) - if !ok { - // TODO (pok) send back the right error - return nil, sql3.NewErrSetExpressionExpected(call.Args[0].Pos().Line, call.Args[0].Pos().Column) - } - call.ResultDataType = parser.NewDataTypeSubtable([]*parser.SubtableColumn{ - { - Name: string(dax.PrimaryKeyFieldName), - DataType: parser.NewDataTypeID(), - }, - { - Name: "timestamp", - DataType: parser.NewDataTypeTimestamp(), - }, - { - Name: "value", - DataType: call.Args[0].DataType(), - }, - }) - return call, nil -} diff --git a/sql3/planner/oppqldistinctscan.go b/sql3/planner/oppqldistinctscan.go index f5f5004da..b1f0cbeff 100644 --- a/sql3/planner/oppqldistinctscan.go +++ b/sql3/planner/oppqldistinctscan.go @@ -71,11 +71,19 @@ func (p *PlanOpPQLDistinctScan) Name() string { return p.tableName } +func (p *PlanOpPQLDistinctScan) IsFilterable() bool { + return true +} + func (p *PlanOpPQLDistinctScan) UpdateFilters(filterCondition types.PlanExpression) (types.PlanOperator, error) { p.filter = filterCondition return p, nil } +func (p *PlanOpPQLDistinctScan) UpdateTimeQuantumFilters(filters ...types.PlanExpression) (types.PlanOperator, error) { + return p, nil +} + func (p *PlanOpPQLDistinctScan) Schema() types.Schema { result := make(types.Schema, 0) diff --git a/sql3/planner/oppqltablescan.go b/sql3/planner/oppqltablescan.go index 431cf30c4..6c876cea3 100644 --- a/sql3/planner/oppqltablescan.go +++ b/sql3/planner/oppqltablescan.go @@ -17,20 +17,22 @@ import ( // PlanOpPQLTableScan plan operator handles a PQL table scan type PlanOpPQLTableScan struct { - planner *ExecutionPlanner - tableName string - columns []string - filter types.PlanExpression - topExpr types.PlanExpression - warnings []string + planner *ExecutionPlanner + tableName string + columns []string + filter types.PlanExpression + timeQuantumFilters []types.PlanExpression + topExpr types.PlanExpression + warnings []string } func NewPlanOpPQLTableScan(p *ExecutionPlanner, tableName string, columns []string) *PlanOpPQLTableScan { return &PlanOpPQLTableScan{ - planner: p, - tableName: tableName, - columns: columns, - warnings: make([]string, 0), + planner: p, + tableName: tableName, + columns: columns, + timeQuantumFilters: make([]types.PlanExpression, 0), + warnings: make([]string, 0), } } @@ -46,6 +48,11 @@ func (p *PlanOpPQLTableScan) Plan() map[string]interface{} { if p.filter != nil { result["filter"] = p.filter.Plan() } + tqfilters := make([]map[string]interface{}, len(p.timeQuantumFilters)) + for i, f := range p.timeQuantumFilters { + tqfilters[i] = f.Plan() + } + result["tqfilters"] = tqfilters result["columns"] = p.columns return result } @@ -66,11 +73,20 @@ func (p *PlanOpPQLTableScan) Name() string { return p.tableName } +func (p *PlanOpPQLTableScan) IsFilterable() bool { + return true +} + func (p *PlanOpPQLTableScan) UpdateFilters(filterCondition types.PlanExpression) (types.PlanOperator, error) { p.filter = filterCondition return p, nil } +func (p *PlanOpPQLTableScan) UpdateTimeQuantumFilters(filters ...types.PlanExpression) (types.PlanOperator, error) { + p.timeQuantumFilters = filters + return p, nil +} + func (p *PlanOpPQLTableScan) Schema() types.Schema { result := make(types.Schema, 0) @@ -101,11 +117,12 @@ func (p *PlanOpPQLTableScan) Children() []types.PlanOperator { func (p *PlanOpPQLTableScan) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { return &tableScanRowIter{ - planner: p.planner, - tableName: p.tableName, - columns: p.columns, - predicate: p.filter, - topExpr: p.topExpr, + planner: p.planner, + tableName: p.tableName, + columns: p.columns, + predicate: p.filter, + timeQuantumFilters: p.timeQuantumFilters, + topExpr: p.topExpr, }, nil } @@ -134,11 +151,12 @@ type targetColumn struct { } type tableScanRowIter struct { - planner *ExecutionPlanner - tableName string - columns []string - predicate types.PlanExpression - topExpr types.PlanExpression + planner *ExecutionPlanner + tableName string + columns []string + predicate types.PlanExpression + timeQuantumFilters []types.PlanExpression + topExpr types.PlanExpression result []pilosa.ExtractedTableColumn rowWidth int @@ -215,12 +233,36 @@ func (i *tableScanRowIter) Next(ctx context.Context) (types.Row, error) { continue } - call.Children = append(call.Children, - &pql.Call{ - Name: "Rows", - Args: map[string]interface{}{"field": c}, - }, - ) + foundInTimeQuantumFilters := false + for _, tqf := range i.timeQuantumFilters { + f, ok := tqf.(*callPlanExpression) + if !ok { + return nil, sql3.NewErrInternalf("unexpected time quantum filter expression type: %T", tqf) + } + // argument 0 should be a column ref + arg, ok := f.args[0].(*qualifiedRefPlanExpression) + if !ok { + return nil, sql3.NewErrInternalf("unexpected time quantum filter argument expression type: %T", f.args[0]) + } + if strings.EqualFold(arg.columnName, c) { + expr, err := i.planner.generatePQLCallFromExpr(ctx, tqf) + if err != nil { + return nil, err + } + + call.Children = append(call.Children, expr) + foundInTimeQuantumFilters = true + } + } + + if !foundInTimeQuantumFilters { + call.Children = append(call.Children, + &pql.Call{ + Name: "Rows", + Args: map[string]interface{}{"field": c}, + }, + ) + } } tbl, err := i.planner.schemaAPI.TableByName(ctx, dax.TableName(i.tableName)) diff --git a/sql3/planner/oprelalias.go b/sql3/planner/oprelalias.go index 2e4d5610c..1691e0a02 100644 --- a/sql3/planner/oprelalias.go +++ b/sql3/planner/oprelalias.go @@ -77,3 +77,37 @@ func (p *PlanOpRelAlias) Warnings() []string { func (p *PlanOpRelAlias) Name() string { return p.alias } + +func (p *PlanOpRelAlias) IsFilterable() bool { + ch, ok := p.ChildOp.(types.FilteredRelation) + if !ok { + return false + } + return ch.IsFilterable() +} + +func (p *PlanOpRelAlias) UpdateFilters(filterCondition types.PlanExpression) (types.PlanOperator, error) { + ch, ok := p.ChildOp.(types.FilteredRelation) + if !ok { + return nil, sql3.NewErrInternalf("childop is not filterable") + } + newChild, err := ch.UpdateFilters(filterCondition) + if err != nil { + return nil, err + } + p.ChildOp = newChild + return p, nil +} + +func (p *PlanOpRelAlias) UpdateTimeQuantumFilters(filters ...types.PlanExpression) (types.PlanOperator, error) { + ch, ok := p.ChildOp.(types.FilteredRelation) + if !ok { + return nil, sql3.NewErrInternalf("childop is not filterable") + } + newChild, err := ch.UpdateTimeQuantumFilters(filters...) + if err != nil { + return nil, err + } + p.ChildOp = newChild + return p, nil +} diff --git a/sql3/planner/opsystemtable.go b/sql3/planner/opsystemtable.go index 91d12c132..2f5c0f91c 100644 --- a/sql3/planner/opsystemtable.go +++ b/sql3/planner/opsystemtable.go @@ -37,7 +37,7 @@ var systemTables = map[string]*systemTable{ schema: types.Schema{ &types.PlannerColumn{ RelationName: fbClusterInfo, - ColumnName: "name", + ColumnName: "id", Type: parser.NewDataTypeString(), }, &types.PlannerColumn{ diff --git a/sql3/planner/optablevaluedfunction.go b/sql3/planner/optablevaluedfunction.go index 1619fcdac..0ce4aa235 100644 --- a/sql3/planner/optablevaluedfunction.go +++ b/sql3/planner/optablevaluedfunction.go @@ -73,5 +73,8 @@ func (p *PlanOpTableValuedFunction) Warnings() []string { var w []string w = append(w, p.warnings...) return w - +} + +func (p *PlanOpTableValuedFunction) Name() string { + return "" } diff --git a/sql3/planner/planoptimizer.go b/sql3/planner/planoptimizer.go index 039930582..55b0bffc5 100644 --- a/sql3/planner/planoptimizer.go +++ b/sql3/planner/planoptimizer.go @@ -50,10 +50,6 @@ var optimizerFunctions = []OptimizerFunc{ // one TableScanOperator, try to use a PQL aggregate operators instead tryToReplaceGroupByWithPQLAggregate, - // if we have a subtable call on a timequantum type - // take the join out and use the appropriate PQL operator instead - tryToRewriteSubtableJoins, - // update the columnIdx for all the qualified references in various operators fixFieldRefs, @@ -216,6 +212,8 @@ func getRelationAliases(n types.PlanOperator, scope *OptimizerScope) (RelationAl inspectErr = aliases.addAlias(node, t) case *PlanOpSubquery: inspectErr = aliases.addAlias(node, t) + case *PlanOpTableValuedFunction: + inspectErr = aliases.addAlias(node, t) default: inspectErr = sql3.NewErrInternalf("unexpected alias child type '%T", node.ChildOp) } @@ -243,7 +241,7 @@ func getRelationAliases(n types.PlanOperator, scope *OptimizerScope) (RelationAl func filterPushdownChildSelector(c ParentContext) bool { switch c.Parent.(type) { case *PlanOpRelAlias: - //definitely don't go any further than alias + //definitely don't go any further than alias as parent return false } return true @@ -356,6 +354,16 @@ func pushdownFiltersToFilterableRelations(ctx context.Context, a *ExecutionPlann // only do this if it is a pql table scan switch rel := tableNode.(type) { + case *PlanOpRelAlias: + switch rel.ChildOp.(type) { + case *PlanOpPQLTableScan: + table = rel + case *PlanOpPQLDistinctScan: + table = rel + default: + return tableNode, true, nil + } + case *PlanOpPQLTableScan: table = rel case *PlanOpPQLDistinctScan: @@ -366,7 +374,7 @@ func pushdownFiltersToFilterableRelations(ctx context.Context, a *ExecutionPlann // is the thing filterable? ft, ok := table.(types.FilteredRelation) - if !ok { + if !ok || !ft.IsFilterable() { return tableNode, true, nil } @@ -377,30 +385,62 @@ func pushdownFiltersToFilterableRelations(ctx context.Context, a *ExecutionPlann } tableFilters := make([]types.PlanExpression, 0) + timeQantumFilters := make([]types.PlanExpression, 0) // can the filters be pushed down? for _, tf := range availableFilters { // try and generate a pql call graph, if we can't we can't push the filter down _, err := a.generatePQLCallFromExpr(ctx, tf) if err == nil { - tableFilters = append(tableFilters, tf) + // is this a time quantum call? + call, ok := tf.(*callPlanExpression) + if ok { + switch strings.ToUpper(call.name) { + case "RANGEQ": + timeQantumFilters = append(timeQantumFilters, tf) + default: + // it's a filter + tableFilters = append(tableFilters, tf) + } + } else { + // it's a filter + tableFilters = append(tableFilters, tf) + } } } // did we end up with any filters? - if len(tableFilters) == 0 { + if len(tableFilters)+len(timeQantumFilters) == 0 { return tableNode, true, nil } - filters.markFiltersHandled(tableFilters...) - - // fix the field refs - tableFilters, _, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, tableNode.Schema(), tableFilters...) - if err != nil { - return nil, true, err + var err error + var newOp types.PlanOperator + //deal with the filters + if len(tableFilters) > 0 { + filters.markFiltersHandled(tableFilters...) + // fix the field refs + tableFilters, _, err = fixFieldRefIndexesOnExpressions(ctx, scope, a, tableNode.Schema(), tableFilters...) + if err != nil { + return nil, true, err + } + newOp, err = ft.UpdateFilters(joinExprsWithAnd(tableFilters...)) + if err != nil { + return nil, true, err + } } - newOp, err := ft.UpdateFilters(joinExprsWithAnd(tableFilters...)) - if err != nil { - return nil, true, err + // deal with any time quantum related filters + if len(timeQantumFilters) > 0 { + filters.markFiltersHandled(timeQantumFilters...) + + timeQantumFilters, _, err = fixFieldRefIndexesOnExpressions(ctx, scope, a, tableNode.Schema(), timeQantumFilters...) + if err != nil { + return nil, true, err + } + + newOp, err = ft.UpdateTimeQuantumFilters(timeQantumFilters...) + if err != nil { + return nil, true, err + } } return newOp, false, nil } @@ -410,6 +450,17 @@ func pushdownFiltersToAboveRelation(ctx context.Context, a *ExecutionPlanner, ta // only do this if it is a pql table scan switch rel := tableNode.(type) { + + case *PlanOpRelAlias: + switch rel.ChildOp.(type) { + case *PlanOpPQLTableScan: + table = rel + case *PlanOpPQLDistinctScan: + table = rel + default: + return tableNode, true, nil + } + case *PlanOpPQLTableScan: table = rel case *PlanOpPQLDistinctScan: @@ -880,71 +931,6 @@ func tryToReplaceGroupByWithPQLGroupBy(ctx context.Context, a *ExecutionPlanner, return n, true, nil } -// the semantic for accessing a timequantum field is to use the subtable() table valued function in a join -// rewrite queries that use this pattern to use the appropriate PQL call -func tryToRewriteSubtableJoins(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { - //bail if there are no joins - joins := getNestedLoopOperators(ctx, a, n, scope) - if len(joins) == 0 { - return n, true, nil - } - - //get the projections, we're going to need them later - projections := getPlanOpProjectionOperators(ctx, a, n, scope) - - return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) { - switch nl := node.(type) { - case *PlanOpNestedLoops: - var tvf *PlanOpTableValuedFunction - // bail if the join does not have a tvf as one of the operators - tvftop, topok := nl.top.(*PlanOpTableValuedFunction) - tvfbottom, bottomok := nl.bottom.(*PlanOpTableValuedFunction) - - //bail if both sides of the join are a tvf - if topok && bottomok { - return nl, true, nil - } - if topok { - tvf = tvftop - } - if bottomok { - tvf = tvfbottom - } - //if tvf == nil, then neither side is a tvf - if tvf == nil { - return nl, true, nil - } - - //check it is the subtable() tvf - tvfCall, ok := tvf.callExpr.(*callPlanExpression) - if !ok { - return nl, true, nil - } - if !strings.EqualFold(tvfCall.name, "subtable") { - return nl, true, nil - } - - // if there is no join condition, it's an extract; replace the 'value' reference - // with a reference with the first argument and remove the join - if nl.cond == nil { - // get the first argument column from the tvf - - // for each of the projection operators, for each of the projections - // transform each of the referenced values with a the first arg - - a.logger.Debugf("%T", projections) - } - - // there is a join condition, make sure it is one that is permissible (range queries only?) - a.logger.Debugf("%T", tvf) - - return nl, true, nil - default: - return nl, true, nil - } - }) -} - func pushdownPQLTop(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { // bail if there are any joins joins, err := hasJoins(ctx, a, n, scope) diff --git a/sql3/planner/types/operator.go b/sql3/planner/types/operator.go index a0e843c03..a0d70cc7e 100644 --- a/sql3/planner/types/operator.go +++ b/sql3/planner/types/operator.go @@ -61,7 +61,9 @@ type Relation interface { // FilteredRelation is an interface to something that can be treated as a relation that can be filtered type FilteredRelation interface { Relation + IsFilterable() bool UpdateFilters(filterCondition PlanExpression) (PlanOperator, error) + UpdateTimeQuantumFilters(filters ...PlanExpression) (PlanOperator, error) } // Schema is the definition a set of columns from each operator diff --git a/sql3/sql_complex_test.go b/sql3/sql_complex_test.go index a25314807..5aebc1bff 100644 --- a/sql3/sql_complex_test.go +++ b/sql3/sql_complex_test.go @@ -276,8 +276,8 @@ func TestPlanner_Show(t *testing.T) { species string cachetype ranked size 1000 speciesids idset cachetype ranked size 1000 speciess stringset cachetype ranked size 1000 - speciesidsq idset timequantum 'YMD' - speciessq stringset timequantum 'YMD' + speciesidsq idsetq timequantum 'YMD' + speciessq stringsetq timequantum 'YMD' specieslen decimal(4) min 0 max 270 ) keypartitions 12 `) @@ -294,7 +294,7 @@ func TestPlanner_Show(t *testing.T) { } if diff := cmp.Diff([][]interface{}{ - {string("create table iris1 (_id id, speciesid id cachetype ranked size 1000, species string cachetype ranked size 1000, speciesids idset cachetype ranked size 1000, speciess stringset cachetype ranked size 1000, speciesidsq idset timequantum 'YMD', speciessq stringset timequantum 'YMD', specieslen decimal(4) min 0 max 270);")}, + {string("create table iris1 (_id id, speciesid id cachetype ranked size 1000, species string cachetype ranked size 1000, speciesids idset cachetype ranked size 1000, speciess stringset cachetype ranked size 1000, speciesidsq idsetq timequantum 'YMD', speciessq stringsetq timequantum 'YMD', specieslen decimal(4) min 0 max 270);")}, }, results); diff != "" { t.Fatal(diff) } @@ -393,13 +393,13 @@ func TestPlanner_CoverCreateTable(t *testing.T) { name: "stringsetcolq", typ: "stringset", constraints: "cachetype lru size 1000 timequantum 'YMD' ttl '24h'", - expErr: "[1:60] 'CACHETYPE' constraint conflicts with 'TIMEQUANTUM'", + expErr: "[1:60] 'TIMEQUANTUM' constraint cannot be applied to a column of type 'stringset'", }, { name: "stringsetcolq", typ: "stringset", constraints: "timequantum 'YMD' ttl '24h' cachetype ranked", - expErr: "[1:60] 'CACHETYPE' constraint conflicts with 'TIMEQUANTUM'", + expErr: "[1:60] 'TIMEQUANTUM' constraint cannot be applied to a column of type 'stringset'", }, } @@ -504,7 +504,7 @@ func TestPlanner_CoverCreateTable(t *testing.T) { }, { name: "stringsetcolq", - typ: "stringset", + typ: "stringsetq", constraints: "timequantum 'YMD' ttl '24h'", expOptions: pilosa.FieldOptions{ Type: "time", @@ -550,7 +550,7 @@ func TestPlanner_CoverCreateTable(t *testing.T) { }, { name: "idsetcolq", - typ: "idset", + typ: "idsetq", constraints: "timequantum 'YMD' ttl '24h'", expOptions: pilosa.FieldOptions{ Type: "time", @@ -717,11 +717,11 @@ func TestPlanner_CreateTable(t *testing.T) { decimalcol decimal(2), stringcol string cachetype ranked size 1000, stringsetcol stringset cachetype lru size 1000, - stringsetcolq stringset timequantum 'YMD' ttl '24h', + stringsetcolq stringsetq timequantum 'YMD' ttl '24h', idcol id cachetype ranked size 1000, idsetcol idset cachetype lru, idsetcolsz idset cachetype lru size 1000, - idsetcolq idset timequantum 'YMD' ttl '24h') keypartitions 12`) + idsetcolq idsetq timequantum 'YMD' ttl '24h') keypartitions 12`) if err != nil { t.Fatal(err) } diff --git a/sql3/test/defs/defs.go b/sql3/test/defs/defs.go index 42e48e9af..18764167d 100644 --- a/sql3/test/defs/defs.go +++ b/sql3/test/defs/defs.go @@ -192,7 +192,8 @@ var TableTests []TableTest = []TableTest{ boolTests, // time quantums - timeQuantumInsertTest, + timeQuantumTest, + timeQuantumQueryTest, // forward-ported SQL1 tests sql1TestsGrouper, diff --git a/sql3/test/defs/defs_join.go b/sql3/test/defs/defs_join.go index 6c0978a82..0fb7e2bf2 100644 --- a/sql3/test/defs/defs_join.go +++ b/sql3/test/defs/defs_join.go @@ -99,7 +99,7 @@ var joinTests = TableTest{ hdr("", fldTypeInt), ), ExpRows: rows( - row(int64(2)), + row(int64(4)), ), Compare: CompareExactOrdered, }, @@ -112,7 +112,7 @@ var joinTests = TableTest{ hdr("", fldTypeInt), ), ExpRows: rows( - row(int64(2)), + row(int64(6)), ), Compare: CompareExactOrdered, }, diff --git a/sql3/test/defs/defs_timequantum.go b/sql3/test/defs/defs_timequantum.go index 6945f5ad6..4440139fe 100644 --- a/sql3/test/defs/defs_timequantum.go +++ b/sql3/test/defs/defs_timequantum.go @@ -1,14 +1,14 @@ package defs -// time quantum insert tests -var timeQuantumInsertTest = TableTest{ +// time quantum tests +var timeQuantumTest = TableTest{ Table: tbl( "time_quantum_insert", srcHdrs( srcHdr("_id", fldTypeID), srcHdr("i1", fldTypeInt, "min 0", "max 1000"), - srcHdr("ss1", fldTypeStringSet, "timequantum 'YMD'"), - srcHdr("ids1", fldTypeIDSet, "timequantum 'YMD'"), + srcHdr("ss1", fldTypeStringSetQ, "timequantum 'YMD'"), + srcHdr("ids1", fldTypeIDSetQ, "timequantum 'YMD'"), ), ), SQLTests: []SQLTest{ @@ -24,7 +24,7 @@ var timeQuantumInsertTest = TableTest{ SQLs: sqls( "insert into time_quantum_insert (_id, i1, ss1, ids1) values (1, 1, {['1']}, {[1]})", ), - ExpErr: "an expression of type 'tuple(stringset)' cannot be assigned to type 'stringset'", + ExpErr: "an expression of type 'tuple(stringset)' cannot be assigned to type 'stringsetq'", }, { SQLs: sqls( @@ -36,12 +36,81 @@ var timeQuantumInsertTest = TableTest{ }, { SQLs: sqls( - "insert into time_quantum_insert (_id, i1, ss1, ids1) values (1, 1, {'2022-01-01T00:00:00Z', ['1']}, {'2022-01-01T00:00:00Z', [1]})", + "insert into time_quantum_insert(_id, i1, ss1, ids1) values (1, 3, ['test1'], [1])", ), ExpHdrs: hdrs(), ExpRows: rows(), Compare: CompareExactUnordered, }, + { + SQLs: sqls( + "insert into time_quantum_insert(_id, i1, ss1, ids1) values (1, 3, {1676649734, ['test2']}, {1676649734, [2]})", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "insert into time_quantum_insert(_id, i1, ss1, ids1) values (1, 3, {'2022-01-01T00:00:00Z', ['test3']}, {'2022-01-01T00:00:00Z', [3]})", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "insert into time_quantum_insert(_id, i1, ss1, ids1) values (1, 3, {'2022-01-02T00:00:00Z', ['test4']}, {'2022-01-01T00:00:00Z', [4]})", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "insert into time_quantum_insert(_id, i1, ss1, ids1) values (1, 3, {'2022-01-03T00:00:00Z', ['test5']}, {'2022-01-01T00:00:00Z', [5]})", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "select a._id, a.ss1 from time_quantum_insert a where rangeq(a.ss1, '2022-01-02T00:00:00Z')", + ), + ExpErr: "'rangeq': count of formal parameters (3) does not match count of actual parameters (2)", + }, + { + SQLs: sqls( + "select a._id, a.ss1 from time_quantum_insert a where rangeq(a.ss1, null, null)", + ), + ExpErr: "alling ranqeq() 'from' and 'to' parameters cannot both be null", + }, + { + SQLs: sqls( + "select a._id, a.ss1 from time_quantum_insert a where rangeq(a.i1, null, null)", + ), + ExpErr: "time quantum expression expected", + }, + { + SQLs: sqls( + "select a._id, a.ss1, rangeq(a.ss1, '2022-01-02T00:00:00Z', null) from time_quantum_insert a", + ), + ExpErr: "calling ranqeq() usage invalid", + }, + { + SQLs: sqls( + "select a._id, a.ss1 from time_quantum_insert a where rangeq(a.ss1, '2022-01-02T00:00:00Z', null)", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + hdr("ss1", fldTypeStringSetQ), + ), + ExpRows: rows( + row(int64(1), []string{"1", "test1", "test2"}), + ), + Compare: CompareExactUnordered, + }, }, } @@ -67,7 +136,7 @@ var timeQuantumQueryTest = TableTest{ SQLTests: []SQLTest{ { SQLs: sqls( - "select _id not like '%f_' from not_like_all_types", + "select _id not like '%f_' from timeQuantumQueryTest", ), ExpErr: "operator 'NOTLIKE' incompatible with type 'id'", }, diff --git a/sql3/test/defs/types.go b/sql3/test/defs/types.go index 666e49209..0ebb57dab 100644 --- a/sql3/test/defs/types.go +++ b/sql3/test/defs/types.go @@ -27,6 +27,10 @@ var ( Type: dax.BaseTypeIDSet, BaseType: dax.BaseTypeIDSet, } + fldTypeIDSetQ featurebase.WireQueryField = featurebase.WireQueryField{ + Type: dax.BaseTypeIDSetQ, + BaseType: dax.BaseTypeIDSetQ, + } fldTypeInt featurebase.WireQueryField = featurebase.WireQueryField{ Type: dax.BaseTypeInt, BaseType: dax.BaseTypeInt, @@ -44,6 +48,10 @@ var ( Type: dax.BaseTypeStringSet, BaseType: dax.BaseTypeStringSet, } + fldTypeStringSetQ featurebase.WireQueryField = featurebase.WireQueryField{ + Type: dax.BaseTypeStringSetQ, + BaseType: dax.BaseTypeStringSetQ, + } fldTypeTimestamp featurebase.WireQueryField = featurebase.WireQueryField{ Type: dax.BaseTypeTimestamp, BaseType: dax.BaseTypeTimestamp,