From b17ee6a2034ddb806b5e87dd5882f9bc8840912b Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula <85502298+pokeeffe-molecula@users.noreply.github.com> Date: Mon, 21 Nov 2022 14:31:45 -0600 Subject: [PATCH] fixed join bugs by fixing query optimizer (fb-1699, fb-1700) (#2307) this commit changes the way the plan is retrieved; implements Stringer on types.PlanExpression in preparation for HAVING support; removes last vestiges internal float64 arithmetic; implements a filter on PlanOpFilter; fixes various bugs in the PlanOptimizer when rewriting qualified references * fixed selects with unqualified identifiers * handle bad and non-existent query param inputs more appropriately * added test coverage for PlanExpression Stringer Co-authored-by: Matthew Jaffee (cherry picked from commit 5f662d2bce6b56173e7ea1791a10333f20e6858b) --- http_handler.go | 23 +++ sql3/parser/astdatatype.go | 5 + sql3/planner/executionplanner.go | 2 - sql3/planner/expression.go | 208 +++++++++++++++++++++- sql3/planner/expression_it_test.go | 84 +++++++++ sql3/planner/expressionagg.go | 64 ++++++- sql3/planner/expressionanalyzer.go | 2 +- sql3/planner/expressionanalyzercall.go | 8 +- sql3/planner/expressiontypes.go | 4 +- sql3/planner/opfilter.go | 34 +++- sql3/planner/opgroupby.go | 7 +- sql3/planner/opnestedloops.go | 17 +- sql3/planner/oppqltablescan.go | 36 ++-- sql3/planner/opprojection.go | 7 +- sql3/planner/opquery.go | 1 + sql3/planner/oprelalias.go | 2 +- sql3/planner/planner.go | 3 - sql3/planner/planoptimizer.go | 228 ++++++++++++------------- sql3/planner/planwalker.go | 47 +++-- sql3/planner/types/planexpression.go | 5 + sql3/sql_complex_test.go | 17 ++ sql3/test/defs/defs_join.go | 49 +++++- sql3/test/defs/defs_unops.go | 10 +- sql3/test/helpers.go | 3 + 24 files changed, 664 insertions(+), 202 deletions(-) create mode 100644 sql3/planner/expression_it_test.go delete mode 100644 sql3/planner/planner.go diff --git a/http_handler.go b/http_handler.go index c3520b4eb..387ab14e2 100644 --- a/http_handler.go +++ b/http_handler.go @@ -1398,6 +1398,17 @@ func (h *Handler) writeBadRequest(w http.ResponseWriter, r *http.Request, err er // handlePostSQL handles /sql requests. func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { + includePlan := false + + includePlanValue := r.URL.Query().Get("plan") + if len(includePlanValue) > 0 { + var err error + includePlan, err = strconv.ParseBool(includePlanValue) + if err != nil { + h.writeBadRequest(w, r, err) + return + } + } b, err := io.ReadAll(r.Body) @@ -1466,6 +1477,17 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { } } + writePlan := func(plan map[string]interface{}) { + if plan != nil && includePlan { + planBytes, err := json.Marshal(plan) + if err != nil { + planBytes = []byte(`"PROBLEM ENCODING QUERY PLAN"`) + } + w.Write([]byte(`,"queryPlan":`)) + w.Write(planBytes) + } + } + // Get a query iterator. iter, err := rootOperator.Iterator(r.Context(), nil) if err != nil { @@ -1529,6 +1551,7 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { writeError(rowErr) writeWarnings(rootOperator.Warnings()) + writePlan(rootOperator.Plan()) } type SQLField struct { diff --git a/sql3/parser/astdatatype.go b/sql3/parser/astdatatype.go index 42879a194..ed13aebb8 100644 --- a/sql3/parser/astdatatype.go +++ b/sql3/parser/astdatatype.go @@ -257,6 +257,11 @@ func FloatToDecimal(v float64) pql.Decimal { return pql.NewDecimal(unscaledValue, int64(scale)) } +func FloatToDecimalWithScale(v float64, s int64) pql.Decimal { + unscaledValue := int64(v * math.Pow(10, float64(s))) + return pql.NewDecimal(unscaledValue, int64(s)) +} + func NumDecimalPlaces(v string) int { i := strings.IndexByte(v, '.') if i > -1 { diff --git a/sql3/planner/executionplanner.go b/sql3/planner/executionplanner.go index 1d166b4d4..5b4781602 100644 --- a/sql3/planner/executionplanner.go +++ b/sql3/planner/executionplanner.go @@ -79,12 +79,10 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen default: return nil, sql3.NewErrInternalf("cannot plan statement: %T", stmt) } - // Optimize the plan. if err == nil { rootOperator, err = p.optimizePlan(ctx, rootOperator) } - return rootOperator, err } diff --git a/sql3/planner/expression.go b/sql3/planner/expression.go index ef6ac4ed9..c42c50349 100644 --- a/sql3/planner/expression.go +++ b/sql3/planner/expression.go @@ -166,6 +166,10 @@ func (n *unaryOpPlanExpression) Type() parser.ExprDataType { return n.resultDataType } +func (n *unaryOpPlanExpression) String() string { + return fmt.Sprintf("%s%s", n.op.String(), n.rhs.String()) +} + func (n *unaryOpPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -233,7 +237,11 @@ func (n *unaryOpPlanExpression) plusWithTypeCheck(rhs interface{}) (interface{}, case *parser.DataTypeDecimal: nr, nrok := rhs.(pql.Decimal) if nrok { - return +(nr.Float64()), nil + val := nr.Value() + if !val.IsInt64() { + return nil, sql3.NewErrInternalf("decimal value overflow: %v", rhs) + } + return pql.NewDecimal(+val.Int64(), nr.Scale), nil } return nil, sql3.NewErrInternalf("unexpected incompatible types '%T", rhs) @@ -266,7 +274,12 @@ func (n *unaryOpPlanExpression) minusWithTypeCheck(rhs interface{}) (interface{} case *parser.DataTypeDecimal: nr, nrok := rhs.(pql.Decimal) if nrok { - return -(nr.Float64()), nil + val := nr.Value() + if !val.IsInt64() { + return nil, sql3.NewErrInternalf("decimal value overflow: %v", rhs) + } + return pql.NewDecimal(-val.Int64(), nr.Scale), nil + } return nil, sql3.NewErrInternalf("unexpected incompatible types '%T", rhs) @@ -644,6 +657,10 @@ func (n *binOpPlanExpression) Type() parser.ExprDataType { return n.resultDataType } +func (n *binOpPlanExpression) String() string { + return fmt.Sprintf("%s%s%s", n.lhs.String(), n.op.String(), n.rhs.String()) +} + func (n *binOpPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -711,6 +728,10 @@ func (n *rangePlanExpression) Type() parser.ExprDataType { return n.resultDataType } +func (n *rangePlanExpression) String() string { + return fmt.Sprintf("between %s and %s", n.lhs.String(), n.rhs.String()) +} + func (n *rangePlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -900,6 +921,24 @@ func (n *casePlanExpression) Type() parser.ExprDataType { return n.resultDataType } +func (n *casePlanExpression) String() string { + var result string + if n.baseExpr != nil { + result = fmt.Sprintf("case %s", n.baseExpr) + } else { + result = "case" + } + for _, blk := range n.blocks { + result += fmt.Sprintf(" %s", blk.String()) + } + if n.elseExpr != nil { + result += fmt.Sprintf(" else %s end", n.elseExpr) + } else { + result += " end" + } + return result +} + func (n *casePlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -919,11 +958,45 @@ func (n *casePlanExpression) Plan() map[string]interface{} { } func (n *casePlanExpression) Children() []types.PlanExpression { - return []types.PlanExpression{} + result := make([]types.PlanExpression, 0) + if n.baseExpr != nil { + result = append(result, n.baseExpr) + } + result = append(result, n.blocks...) + if n.elseExpr != nil { + result = append(result, n.elseExpr) + } + return result } func (n *casePlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) { - return n, nil + currentLen := 0 + if n.baseExpr != nil { + currentLen += 1 + } + currentLen += len(n.blocks) + if n.elseExpr != nil { + currentLen += 1 + } + if len(children) != currentLen { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + + offset := 0 + var newBaseExpr types.PlanExpression + if n.baseExpr != nil { + newBaseExpr = children[offset] + offset += 1 + } + newBlocks := make([]types.PlanExpression, len(n.blocks)) + copy(newBlocks[0:], children[offset:offset+len(n.blocks)]) + offset += len(n.blocks) + + var newElseExpr types.PlanExpression + if n.elseExpr != nil { + newElseExpr = children[offset] + } + return newCasePlanExpression(newBaseExpr, newBlocks, newElseExpr, n.resultDataType), nil } // caseBlockPlanExpression is for case blocks @@ -947,6 +1020,10 @@ func (n *caseBlockPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeBool() } +func (n *caseBlockPlanExpression) String() string { + return fmt.Sprintf("when %s then %s end", n.condition.String(), n.body.String()) +} + func (n *caseBlockPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -964,7 +1041,10 @@ func (n *caseBlockPlanExpression) Children() []types.PlanExpression { } func (n *caseBlockPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) { - return n, nil + if len(children) != 2 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return newCaseBlockPlanExpression(children[0], children[1]), nil } // subqueryPlanExpression is a select statement (when used in an expression) @@ -1009,6 +1089,10 @@ func (n *subqueryPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeBool() } +func (n *subqueryPlanExpression) String() string { + return n.op.String() +} + func (n *subqueryPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1110,6 +1194,13 @@ func (n *betweenOpPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeBool() } +func (n *betweenOpPlanExpression) String() string { + if n.op == parser.BETWEEN { + return fmt.Sprintf("between %s and %s", n.lhs.String(), n.rhs.String()) + } + return fmt.Sprintf("not between %s and %s", n.lhs.String(), n.rhs.String()) +} + func (n *betweenOpPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1127,7 +1218,7 @@ func (n *betweenOpPlanExpression) Children() []types.PlanExpression { } func (n *betweenOpPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) { - if len(children) != 1 { + if len(children) != 2 { return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) } return newBetweenOpPlanExpression(children[0], n.op, children[1]), nil @@ -1317,6 +1408,17 @@ func (n *inOpPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeBool() } +func (n *inOpPlanExpression) String() string { + s := n.lhs.String() + if n.op == parser.NOTIN { + s += " not " + } + s += " in (" + s += n.rhs.String() + s += ")" + return s +} + func (n *inOpPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1335,7 +1437,7 @@ func (n *inOpPlanExpression) Children() []types.PlanExpression { } func (n *inOpPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) { - if len(children) != 1 { + if len(children) != 2 { return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) } return newInOpPlanExpression(children[0], n.op, children[1]), nil @@ -1375,6 +1477,17 @@ func (n *callPlanExpression) Type() parser.ExprDataType { return n.dataType } +func (n *callPlanExpression) String() string { + args := "" + for idx, arg := range n.args { + if idx > 0 { + args += ", " + } + args += arg.String() + } + return fmt.Sprintf("%s(%s)", n.name, args) +} + func (n *callPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1427,6 +1540,10 @@ func (n *aliasPlanExpression) Type() parser.ExprDataType { return n.expr.Type() } +func (n *aliasPlanExpression) String() string { + return fmt.Sprintf("%s as %s", n.expr.String(), n.aliasName) +} + func (n *aliasPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1513,6 +1630,13 @@ func (n *qualifiedRefPlanExpression) Type() parser.ExprDataType { return n.dataType } +func (n *qualifiedRefPlanExpression) String() string { + if len(n.tableName) > 0 { + return fmt.Sprintf("%s.%s", n.tableName, n.columnName) + } + return n.columnName +} + func (n *qualifiedRefPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1571,6 +1695,10 @@ func (n *variableRefPlanExpression) Type() parser.ExprDataType { return n.dataType } +func (n *variableRefPlanExpression) String() string { + return fmt.Sprintf("@%s", n.name) +} + func (n *variableRefPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1602,6 +1730,10 @@ func (n *nullLiteralPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeVoid() } +func (n *nullLiteralPlanExpression) String() string { + return "null" +} + func (n *nullLiteralPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1636,6 +1768,10 @@ func (n *intLiteralPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeInt() } +func (n *intLiteralPlanExpression) String() string { + return fmt.Sprintf("%d", n.value) +} + func (n *intLiteralPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1678,6 +1814,10 @@ func (n *floatLiteralPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeDecimal(int64(scale)) } +func (n *floatLiteralPlanExpression) String() string { + return n.value +} + func (n *floatLiteralPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1713,6 +1853,10 @@ func (n *boolLiteralPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeBool() } +func (n *boolLiteralPlanExpression) String() string { + return fmt.Sprintf("%v", n.value) +} + func (n *boolLiteralPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1748,6 +1892,10 @@ func (n *dateLiteralPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeTimestamp() } +func (n *dateLiteralPlanExpression) String() string { + return n.value.Format(time.RFC3339Nano) +} + func (n *dateLiteralPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -1783,6 +1931,10 @@ func (n *stringLiteralPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeString() } +func (n *stringLiteralPlanExpression) String() string { + return fmt.Sprintf("'%s'", n.value) +} + func (n *stringLiteralPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -2006,6 +2158,10 @@ func (n *castPlanExpression) Type() parser.ExprDataType { return n.targetType } +func (n *castPlanExpression) String() string { + return fmt.Sprintf("cast(%s as %s)", n.lhs.String(), n.targetType.TypeName()) +} + func (n *castPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -2046,6 +2202,17 @@ func (n *exprListPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeVoid() } +func (n *exprListPlanExpression) String() string { + var s string + for idx, expr := range n.exprs { + if idx > 0 { + s += ", " + } + s += expr.String() + } + return fmt.Sprintf("(%s)", s) +} + func (n *exprListPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -2062,7 +2229,10 @@ func (n *exprListPlanExpression) Children() []types.PlanExpression { } func (n *exprListPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) { - return n, nil + if len(children) != len(n.exprs) { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return newExprListExpression(children), nil } // exprSetLiteralPlanExpression is a set literal @@ -2122,6 +2292,17 @@ func (n *exprSetLiteralPlanExpression) Type() parser.ExprDataType { return n.dataType } +func (n *exprSetLiteralPlanExpression) String() string { + var members string + for idx, m := range n.members { + if idx > 0 { + members += ", " + } + members += m.String() + } + return fmt.Sprintf("[%s]", members) +} + func (n *exprSetLiteralPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -2192,6 +2373,17 @@ func (n *exprTupleLiteralPlanExpression) Type() parser.ExprDataType { return n.dataType } +func (n *exprTupleLiteralPlanExpression) String() string { + members := "" + for idx, m := range n.members { + if idx > 0 { + members += ", " + } + members += m.String() + } + return fmt.Sprintf("{%s}", members) +} + func (n *exprTupleLiteralPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) diff --git a/sql3/planner/expression_it_test.go b/sql3/planner/expression_it_test.go new file mode 100644 index 000000000..41f1c4ac8 --- /dev/null +++ b/sql3/planner/expression_it_test.go @@ -0,0 +1,84 @@ +package planner + +import ( + "testing" + "time" + + "github.com/molecula/featurebase/v3/sql3/parser" + "github.com/molecula/featurebase/v3/sql3/planner/types" + "github.com/stretchr/testify/assert" +) + +func TestExpressions(t *testing.T) { + + t.Run("StringerTest", func(t *testing.T) { + + uop := newUnaryOpPlanExpression(parser.PLUS, newIntLiteralPlanExpression(10), parser.NewDataTypeInt()) + assert.Equal(t, uop.String(), "+10") + + bop := newBinOpPlanExpression(newIntLiteralPlanExpression(10), parser.PLUS, newIntLiteralPlanExpression(20), parser.NewDataTypeInt()) + assert.Equal(t, bop.String(), "10+20") + + rop := newRangeOpPlanExpression(newIntLiteralPlanExpression(10), newIntLiteralPlanExpression(20), parser.NewDataTypeInt()) + assert.Equal(t, rop.String(), "between 10 and 20") + + cop := newCasePlanExpression(newStringLiteralPlanExpression("foo"), + []types.PlanExpression{ + newCaseBlockPlanExpression(newStringLiteralPlanExpression("20"), newIntLiteralPlanExpression(20)), + newCaseBlockPlanExpression(newStringLiteralPlanExpression("30"), newIntLiteralPlanExpression(20)), + }, + newIntLiteralPlanExpression(20), parser.NewDataTypeInt()) + assert.Equal(t, cop.String(), "case 'foo' when '20' then 20 end when '30' then 20 end else 20 end") + + bwop := newBetweenOpPlanExpression(newIntLiteralPlanExpression(10), parser.BETWEEN, newIntLiteralPlanExpression(20)) + assert.Equal(t, bwop.String(), "between 10 and 20") + + iop := newInOpPlanExpression(newIntLiteralPlanExpression(10), parser.IN, newIntLiteralPlanExpression(20)) + assert.Equal(t, iop.String(), "10 in (20)") + + callop := newCallPlanExpression("foo", []types.PlanExpression{newIntLiteralPlanExpression(10)}, parser.NewDataTypeInt()) + assert.Equal(t, callop.String(), "foo(10)") + + alop := newAliasPlanExpression("frobny", newIntLiteralPlanExpression(10)) + assert.Equal(t, alop.String(), "10 as frobny") + + qrop := newQualifiedRefPlanExpression("foo", "bar", 1, parser.NewDataTypeInt()) + assert.Equal(t, qrop.String(), "foo.bar") + + vop := newVariableRefPlanExpression("foo", 1, parser.NewDataTypeInt()) + assert.Equal(t, vop.String(), "@foo") + + nulop := newNullLiteralPlanExpression() + assert.Equal(t, nulop.String(), "null") + + ilop := newIntLiteralPlanExpression(10) + assert.Equal(t, ilop.String(), "10") + + flop := newFloatLiteralPlanExpression("12.3456") + assert.Equal(t, flop.String(), "12.3456") + + blop := newBoolLiteralPlanExpression(false) + assert.Equal(t, blop.String(), "false") + + tm, _ := time.ParseInLocation(time.RFC3339, "2012-11-01T22:08:41+00:00", time.UTC) + dlop := newDateLiteralPlanExpression(tm) + assert.Equal(t, dlop.String(), "2012-11-01T22:08:41Z") + + slop := newStringLiteralPlanExpression("foo") + assert.Equal(t, slop.String(), "'foo'") + + ctop := newCastPlanExpression(newIntLiteralPlanExpression(10), parser.NewDataTypeString()) + assert.Equal(t, ctop.String(), "cast(10 as STRING)") + + elop := newExprListExpression([]types.PlanExpression{newStringLiteralPlanExpression("foo"), newStringLiteralPlanExpression("bar")}) + assert.Equal(t, elop.String(), "('foo', 'bar')") + + stlop := newExprSetLiteralPlanExpression([]types.PlanExpression{newStringLiteralPlanExpression("foo"), newStringLiteralPlanExpression("bar")}, parser.NewDataTypeString()) + assert.Equal(t, stlop.String(), "['foo', 'bar']") + + tplop := newExprTupleLiteralPlanExpression([]types.PlanExpression{newStringLiteralPlanExpression("foo"), newStringLiteralPlanExpression("bar")}, parser.NewDataTypeString()) + assert.Equal(t, tplop.String(), "{'foo', 'bar'}") + + }) + +} diff --git a/sql3/planner/expressionagg.go b/sql3/planner/expressionagg.go index 15fbcba68..d4bf9899d 100644 --- a/sql3/planner/expressionagg.go +++ b/sql3/planner/expressionagg.go @@ -120,6 +120,10 @@ func (n *countPlanExpression) Type() parser.ExprDataType { return n.returnDataType } +func (n *countPlanExpression) String() string { + return fmt.Sprintf("count(%s)", n.arg.String()) +} + func (n *countPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -179,6 +183,10 @@ func (n *countDistinctPlanExpression) Type() parser.ExprDataType { return n.returnDataType } +func (n *countDistinctPlanExpression) String() string { + return fmt.Sprintf("count(distinct %s)", n.arg.String()) +} + func (n *countDistinctPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -197,13 +205,12 @@ func (n *countDistinctPlanExpression) WithChildren(children ...types.PlanExpress // aggregator for the SUM function type aggregateSum struct { - sum float64 + sum interface{} expr types.PlanExpression } func NewAggSumBuffer(child types.PlanExpression) *aggregateSum { return &aggregateSum{ - sum: float64(0), expr: child, } } @@ -230,7 +237,17 @@ func (m *aggregateSum) Update(ctx context.Context, row types.Row) error { if !ok { return sql3.NewErrInternalf("unexpected type conversion '%T'", v) } - m.sum += val.Float64() + var dsum pql.Decimal + if m.sum != nil { + dsum, ok = m.sum.(pql.Decimal) + if !ok { + return sql3.NewErrInternalf("unexpected type conversion '%T'", m.sum) + } + } else { + dsum = pql.NewDecimal(0, dataType.Scale) + } + dsum = pql.AddDecimal(dsum, val) + m.sum = dsum default: return sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", dataType) } @@ -238,7 +255,17 @@ func (m *aggregateSum) Update(ctx context.Context, row types.Row) error { } func (m *aggregateSum) Eval(ctx context.Context) (interface{}, error) { - return m.sum, nil + switch m.expr.Type().(type) { + case *parser.DataTypeDecimal: + dsum, ok := m.sum.(pql.Decimal) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type conversion '%T'", m.sum) + } + return dsum, nil + default: + return nil, sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", m.expr.Type()) + } + } // sumPlanExpression handles SUM() @@ -284,6 +311,10 @@ func (n *sumPlanExpression) Type() parser.ExprDataType { return n.returnDataType } +func (n *sumPlanExpression) String() string { + return fmt.Sprintf("sum(%s)", n.arg.String()) +} + func (n *sumPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -293,11 +324,16 @@ func (n *sumPlanExpression) Plan() map[string]interface{} { } func (n *sumPlanExpression) Children() []types.PlanExpression { - return nil + return []types.PlanExpression{ + n.arg, + } } func (n *sumPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) { - return n, nil + if len(children) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return newSumPlanExpression(children[0], n.returnDataType), nil } // aggregator for AVG @@ -387,6 +423,10 @@ func (n *avgPlanExpression) Type() parser.ExprDataType { return n.returnDataType } +func (n *avgPlanExpression) String() string { + return fmt.Sprintf("avg(%s)", n.arg.String()) +} + func (n *avgPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -480,6 +520,10 @@ func (n *minPlanExpression) Type() parser.ExprDataType { return n.returnDataType } +func (n *minPlanExpression) String() string { + return fmt.Sprintf("min(%s)", n.arg.String()) +} + func (n *minPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -573,6 +617,10 @@ func (n *maxPlanExpression) Type() parser.ExprDataType { return n.returnDataType } +func (n *maxPlanExpression) String() string { + return fmt.Sprintf("max(%s)", n.arg.String()) +} + func (n *maxPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) @@ -636,6 +684,10 @@ func (n *percentilePlanExpression) Type() parser.ExprDataType { return n.returnDataType } +func (n *percentilePlanExpression) String() string { + return fmt.Sprintf("percentile(%s)", n.arg.String()) +} + func (n *percentilePlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) diff --git a/sql3/planner/expressionanalyzer.go b/sql3/planner/expressionanalyzer.go index ca86d78bb..d212ddf22 100644 --- a/sql3/planner/expressionanalyzer.go +++ b/sql3/planner/expressionanalyzer.go @@ -337,7 +337,7 @@ func (p *ExecutionPlanner) analyzeUnaryExpression(expr *parser.UnaryExpr, scope } if typeIsInteger(x.DataType()) { expr.ResultDataType = parser.NewDataTypeInt() - } else if typeIsFloat(x.DataType()) { + } else if typeIsDecimal(x.DataType()) { fd, ok := x.DataType().(*parser.DataTypeDecimal) if !ok { return nil, sql3.NewErrInternalf("unexpected data type") diff --git a/sql3/planner/expressionanalyzercall.go b/sql3/planner/expressionanalyzercall.go index 3951f16db..ff6489ca0 100644 --- a/sql3/planner/expressionanalyzercall.go +++ b/sql3/planner/expressionanalyzercall.go @@ -67,7 +67,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser } //make sure the ref is sum-able - if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType())) { + if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType())) { return nil, sql3.NewErrIntOrDecimalExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column) } @@ -95,7 +95,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser } //make sure the ref is avg-able - if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType())) { + if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType())) { return nil, sql3.NewErrIntOrDecimalExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column) } @@ -123,7 +123,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser } //make sure the ref is percentilable-able - if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType()) || typeIsTimestamp(ref.DataType())) { + if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType()) || typeIsTimestamp(ref.DataType())) { return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column) } @@ -164,7 +164,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser } // make sure the ref is min/max-able - if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType()) || typeIsTimestamp(ref.DataType())) { + if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType()) || typeIsTimestamp(ref.DataType())) { return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column) } diff --git a/sql3/planner/expressiontypes.go b/sql3/planner/expressiontypes.go index 118b2425e..5554cd423 100644 --- a/sql3/planner/expressiontypes.go +++ b/sql3/planner/expressiontypes.go @@ -481,8 +481,8 @@ func typeIsTimestamp(testType parser.ExprDataType) bool { } } -// returns true if the type is a float -func typeIsFloat(testType parser.ExprDataType) bool { +// returns true if the type is a decimal +func typeIsDecimal(testType parser.ExprDataType) bool { switch testType.(type) { case *parser.DataTypeDecimal: return true diff --git a/sql3/planner/opfilter.go b/sql3/planner/opfilter.go index 65f71888f..69b042b82 100644 --- a/sql3/planner/opfilter.go +++ b/sql3/planner/opfilter.go @@ -61,6 +61,7 @@ func (p *PlanOpFilter) Plan() map[string]interface{} { ps = append(ps, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeName())) } result["_schema"] = ps + result["predicate"] = p.Predicate.Plan() result["child"] = p.ChildOp.Plan() return result } @@ -77,6 +78,23 @@ func (p *PlanOpFilter) Warnings() []string { return p.warnings } +func (p *PlanOpFilter) Expressions() []types.PlanExpression { + if p.Predicate != nil { + return []types.PlanExpression{ + p.Predicate, + } + } + return []types.PlanExpression{} +} + +func (p *PlanOpFilter) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) { + if len(exprs) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs)) + } + p.Predicate = exprs[0] + return p, nil +} + type filterIterator struct { predicate types.PlanExpression child types.RowIterator @@ -92,6 +110,18 @@ func newFilterIterator(ctx context.Context, predicate types.PlanExpression, chil } func (i *filterIterator) Next(ctx context.Context) (types.Row, error) { - //TODO (pok) - actually implement the filter - return i.child.Next(ctx) + for { + row, err := i.child.Next(ctx) + if err != nil { + return nil, err + } + matches, err := conditionIsTrue(ctx, row, i.predicate) + if err != nil { + return nil, err + } + if !matches { + continue + } + return row, nil + } } diff --git a/sql3/planner/opgroupby.go b/sql3/planner/opgroupby.go index c5e796350..5d3e5c609 100644 --- a/sql3/planner/opgroupby.go +++ b/sql3/planner/opgroupby.go @@ -49,7 +49,7 @@ func (p *PlanOpGroupBy) Schema() types.Schema { offset := len(p.GroupByExprs) for idx, agg := range p.Aggregates { s := &types.PlannerColumn{ - ColumnName: "", + ColumnName: agg.String(), RelationName: "", Type: agg.Type(), } @@ -86,7 +86,7 @@ func (p *PlanOpGroupBy) WithChildren(children ...types.PlanOperator) (types.Plan func (p *PlanOpGroupBy) Expressions() []types.PlanExpression { result := []types.PlanExpression{} - result = append(result, p.GroupByExprs...) + result = append(result, p.Aggregates...) return result } @@ -94,8 +94,7 @@ func (p *PlanOpGroupBy) WithUpdatedExpressions(exprs ...types.PlanExpression) (t if len(exprs) != 1 { return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs)) } - p.GroupByExprs = exprs - return p, nil + return NewPlanOpGroupBy(exprs, p.GroupByExprs, p.ChildOp), nil } func (p *PlanOpGroupBy) Plan() map[string]interface{} { diff --git a/sql3/planner/opnestedloops.go b/sql3/planner/opnestedloops.go index b40f7098e..ee40c4ef7 100644 --- a/sql3/planner/opnestedloops.go +++ b/sql3/planner/opnestedloops.go @@ -38,6 +38,7 @@ func (p *PlanOpNestedLoops) Plan() map[string]interface{} { result["_schema"] = ps result["top"] = p.top.Plan() result["bottom"] = p.bottom.Plan() + result["condition"] = p.cond.Plan() return result } @@ -124,8 +125,6 @@ type nestedLoopsIter struct { rowSize int originalRow types.Row - - bottomRows RowCache } func newNestedLoopsIter(ctx context.Context, jt joinType, top types.RowIterator, bottom types.RowIterable, scopeRow types.Row, joinCondition types.PlanExpression, rowWidth int, originalRow types.Row) *nestedLoopsIter { @@ -136,7 +135,6 @@ func newNestedLoopsIter(ctx context.Context, jt joinType, top types.RowIterator, cond: joinCondition, rowSize: rowWidth, originalRow: originalRow, - bottomRows: newInMemoryRowCache(), ctx: ctx, } } @@ -153,11 +151,14 @@ func (i *nestedLoopsIter) loadTop(ctx context.Context) error { i.topRow = i.originalRow.Append(r) i.foundMatch = false + //DEBUG log.Printf("top row %v", i.topRow) + return nil } func (i *nestedLoopsIter) loadBottom(ctx context.Context) (row types.Row, err error) { if i.bottom == nil { + // DEBUG log.Printf("bottom row initializing iterator...") var iter types.RowIterator iter, err = i.bottomProvider.Iterator(ctx, i.topRow) if err != nil { @@ -169,16 +170,15 @@ func (i *nestedLoopsIter) loadBottom(ctx context.Context) (row types.Row, err er rightRow, err := i.bottom.Next(ctx) if err != nil { if err == types.ErrNoMoreRows { + // DEBUG log.Printf("bottom end of rows") i.bottom = nil i.topRow = nil return nil, types.ErrNoMoreRows } return nil, err } - err = i.bottomRows.Add(rightRow) - if err != nil { - return nil, err - } + + //DEBUG log.Printf("bottom row %v", rightRow) return rightRow, nil } @@ -247,7 +247,8 @@ func (i *nestedLoopsIter) Next(ctx context.Context) (types.Row, error) { i.foundMatch = true - //DEBUG log.Printf("Join result %v", row) + // DEBUG log.Printf("join result %v", row) + return row, nil } } diff --git a/sql3/planner/oppqltablescan.go b/sql3/planner/oppqltablescan.go index 00e0b5920..f267e1fd7 100644 --- a/sql3/planner/oppqltablescan.go +++ b/sql3/planner/oppqltablescan.go @@ -85,12 +85,17 @@ func (p *PlanOpPQLTableScan) Schema() types.Schema { return result } - for _, col := range table.Fields { - result = append(result, &types.PlannerColumn{ - ColumnName: col.Name, - RelationName: p.tableName, - Type: fieldSQLDataType(col), - }) + for _, col := range p.columns { + for _, fld := range table.Fields { + if strings.EqualFold(fld.Name, col) { + result = append(result, &types.PlannerColumn{ + ColumnName: fld.Name, + RelationName: p.tableName, + Type: fieldSQLDataType(fld), + }) + break + } + } } return result } @@ -149,15 +154,20 @@ func (i *tableScanRowIter) Next(ctx context.Context) (types.Row, error) { } return nil, err } - i.rowWidth = len(table.Fields) + i.rowWidth = len(i.columns) i.columnMap = make(map[string]*targetColumn) - for idx, fld := range table.Fields { - i.columnMap[fld.Name] = &targetColumn{ - columnIdx: idx, - srcColumnIdx: -1, - columnName: fld.Name, - dataType: fieldSQLDataType(fld), + for idx, col := range i.columns { + for _, fld := range table.Fields { + if strings.EqualFold(col, fld.Name) { + i.columnMap[fld.Name] = &targetColumn{ + columnIdx: idx, + srcColumnIdx: -1, + columnName: fld.Name, + dataType: fieldSQLDataType(fld), + } + break + } } } diff --git a/sql3/planner/opprojection.go b/sql3/planner/opprojection.go index cf0cfc347..2c0dbe3e6 100644 --- a/sql3/planner/opprojection.go +++ b/sql3/planner/opprojection.go @@ -60,12 +60,12 @@ func (p *PlanOpProjection) WithChildren(children ...types.PlanOperator) (types.P func (p *PlanOpProjection) Plan() map[string]interface{} { result := make(map[string]interface{}) - result["_op"] = fmt.Sprintf("%T", p) + result["__op"] = fmt.Sprintf("%T", p) sc := make([]string, 0) for _, e := range p.Schema() { sc = append(sc, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeName())) } - result["_schema"] = sc + result["__schema"] = sc result["child"] = p.ChildOp.Plan() @@ -73,7 +73,7 @@ func (p *PlanOpProjection) Plan() map[string]interface{} { for _, e := range p.Projections { ps = append(ps, e.Plan()) } - result["projections"] = ps + result["_projections"] = ps return result } @@ -100,7 +100,6 @@ func ExpressionToColumn(e types.PlanExpression) *types.PlannerColumn { if n, ok := e.(types.IdentifiableByName); ok { name = n.Name() } else { - //TODO(pok) - implement this name = "" //e.String() } diff --git a/sql3/planner/opquery.go b/sql3/planner/opquery.go index 89b6f091c..112f1189b 100644 --- a/sql3/planner/opquery.go +++ b/sql3/planner/opquery.go @@ -28,6 +28,7 @@ func NewPlanOpQuery(child types.PlanOperator, sql string) *PlanOpQuery { return &PlanOpQuery{ ChildOp: child, warnings: make([]string, 0), + sql: sql, } } diff --git a/sql3/planner/oprelalias.go b/sql3/planner/oprelalias.go index 7aa13ee9a..3ced6a84f 100644 --- a/sql3/planner/oprelalias.go +++ b/sql3/planner/oprelalias.go @@ -28,7 +28,7 @@ func NewPlanOpRelAlias(alias string, child types.PlanOperator) *PlanOpRelAlias { func (p *PlanOpRelAlias) Schema() types.Schema { schema := p.ChildOp.Schema() for _, s := range schema { - s.RelationName = p.alias + s.AliasName = p.alias } return schema } diff --git a/sql3/planner/planner.go b/sql3/planner/planner.go deleted file mode 100644 index 80a97bd88..000000000 --- a/sql3/planner/planner.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package planner contains everything required to build a query plan from a SQL -// statement. -package planner diff --git a/sql3/planner/planoptimizer.go b/sql3/planner/planoptimizer.go index 9b29955fc..b5b3df538 100644 --- a/sql3/planner/planoptimizer.go +++ b/sql3/planner/planoptimizer.go @@ -8,10 +8,9 @@ import ( "reflect" "strings" - "github.com/featurebasedb/featurebase/v3/errors" - "github.com/featurebasedb/featurebase/v3/sql3" - "github.com/featurebasedb/featurebase/v3/sql3/parser" - "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "github.com/molecula/featurebase/v3/sql3" + "github.com/molecula/featurebase/v3/sql3/parser" + "github.com/molecula/featurebase/v3/sql3/planner/types" ) //TODO(pok) push order by down as far as possible @@ -41,15 +40,12 @@ var optimizerFunctions = []OptimizerFunc{ // take the join out and use the appropriate PQL operator instead tryToRewriteSubtableJoins, + // update the columnIdx for all the qualified references in various operators + fixFieldRefs, + // update the columnIdx for all the references in the projections // based on the child operator for a projection - fixGroupByProjections, - - // update the columnIdx for all the references in joins - fixJoinFieldRefs, - - // update the columnIdx for all the references in group bys - fixGroupByFieldRefs, + fixProjectionReferences, // if the query has one TableScanOperator then push the top // expression down into that operator @@ -64,12 +60,12 @@ type OptimizerScope struct { // optimizePlan takes a plan from the compiler and executes a series of transforms on it to optimize it func (p *ExecutionPlanner) optimizePlan(ctx context.Context, plan types.PlanOperator) (types.PlanOperator, error) { - //log.Println("================================================================================") - //log.Println("plan pre-optimzation") - //jplan := plan.Plan() - //a, _ := json.MarshalIndent(jplan, "", " ") - //log.Println(string(a)) - //log.Println("--------------------------------------------------------------------------------") + // log.Println("================================================================================") + // log.Println("plan pre-optimzation") + // jplan := plan.Plan() + // a, _ := json.MarshalIndent(jplan, "", " ") + // log.Println(string(a)) + // log.Println("--------------------------------------------------------------------------------") var err error var result = plan @@ -79,6 +75,14 @@ func (p *ExecutionPlanner) optimizePlan(ctx context.Context, plan types.PlanOper return nil, err } } + + // log.Println("================================================================================") + // log.Println("plan ppst-optimzation") + // jplan = plan.Plan() + // a, _ = json.MarshalIndent(jplan, "", " ") + // log.Println(string(a)) + // log.Println("--------------------------------------------------------------------------------") + return result, nil } @@ -297,6 +301,7 @@ func pushdownFiltersToFilterableRelations(ctx context.Context, a *ExecutionPlann } filters.markFiltersHandled(tableFilters...) + // fix the field refs tableFilters, _, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, tableNode.Schema(), tableFilters...) if err != nil { return nil, true, err @@ -320,6 +325,7 @@ func pushdownFiltersToAboveRelation(ctx context.Context, a *ExecutionPlanner, ta if tableFilters := filters.availableFiltersForTable(table.Name()); len(tableFilters) > 0 { filters.markFiltersHandled(tableFilters...) + // fix the field refs handled, _, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, tableNode.Schema(), tableFilters...) if err != nil { return nil, true, err @@ -355,24 +361,16 @@ func pushdownFilters(ctx context.Context, a *ExecutionPlanner, n types.PlanOpera if err != nil { return nil, true, err } - n, sameFix, err := fixFieldRefIndexesForOperator(ctx, a, n, scope) - if err != nil { - return nil, true, err - } - return n, samePred && sameFix, nil + return n, samePred, nil case *PlanOpRelAlias, *PlanOpPQLTableScan: n, samePred, err := pushdownFiltersToFilterableRelations(ctx, a, node, scope, filters, tableAliases) if err != nil { return nil, true, err } - n, sameFix, err := fixFieldRefIndexesForOperator(ctx, a, n, scope) - if err != nil { - return nil, true, err - } - return n, samePred && sameFix, nil + return n, samePred, nil default: - return fixFieldRefIndexesForOperator(ctx, a, node, scope) + return node, true, nil } }) } @@ -388,26 +386,18 @@ func pushdownFilters(ctx context.Context, a *ExecutionPlanner, n types.PlanOpera if same { return n, true, nil } - n, _, err = fixFieldRefIndexesForOperator(ctx, a, n, scope) - if err != nil { - return nil, true, err - } return n, false, nil case *PlanOpRelAlias, *PlanOpPQLTableScan: - table, same, err := pushdownFiltersToAboveRelation(ctx, a, node, scope, filters) + _, same, err := pushdownFiltersToAboveRelation(ctx, a, node, scope, filters) if err != nil { return nil, true, err } if same { return node, true, nil } - node, _, err = fixFieldRefIndexesForOperator(ctx, a, table, scope) - if err != nil { - return nil, true, err - } return node, false, nil default: - return fixFieldRefIndexesForOperator(ctx, a, node, scope) + return node, true, nil } }) } @@ -730,23 +720,30 @@ func areAggregablesEqual(lhs types.Aggregable, rhs types.Aggregable) bool { return false } -func fixGroupByProjections(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { +// fixes references for a projection op depending on child +func fixProjectionReferences(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) { switch n := node.(type) { case *PlanOpProjection: switch childOp := n.ChildOp.(type) { + case *PlanOpGroupBy: //PlanOpGroupBy's iterator returns group by exprs, then aggregates in the order they appear - + childSchema := childOp.Schema() for idx, pj := range n.Projections { expr, _, err := TransformExpr(pj, func(e types.PlanExpression) (types.PlanExpression, bool, error) { - switch e.(type) { + switch thisAggregate := e.(type) { case types.Aggregable: // if we have a Aggregable, the AggExpression() will be a qualified ref // given we are in the context of a PlanOpProjection with a PlanOpGroupBy // we can use the ordinal position of the projection as the column index - ae := newQualifiedRefPlanExpression("", "", idx, e.Type()) - return ae, false, nil + for idx, sc := range childSchema { + if strings.EqualFold(thisAggregate.String(), sc.ColumnName) { + ae := newQualifiedRefPlanExpression("", "", idx, e.Type()) + return ae, false, nil + } + } + return nil, true, sql3.NewErrColumnNotFound(0, 0, thisAggregate.String()) default: return e, true, nil } @@ -893,40 +890,75 @@ func fixGroupByProjections(ctx context.Context, a *ExecutionPlanner, n types.Pla n.Projections[idx] = expr } return n, false, nil + + // everything else that can be a child of projection + case *PlanOpRelAlias, *PlanOpFilter, *PlanOpPQLTableScan, *PlanOpNestedLoops: + exprs, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, childOp.Schema(), n.Projections...) + if err != nil { + return n, true, err + } + n.Projections = exprs + return n, same, err + + default: + return n, true, nil } - return n, true, nil + default: return n, true, nil } }) } -func fixJoinFieldRefs(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { +func fixFieldRefs(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) { - switch n := node.(type) { + switch thisNode := node.(type) { + case *PlanOpFilter: + schema := thisNode.Schema() + expressions := thisNode.Expressions() + fixed, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, expressions...) + if err != nil { + return nil, true, err + } + newNode, err := thisNode.WithUpdatedExpressions(fixed...) + if err != nil { + return nil, true, err + } + return newNode, same, nil + case *PlanOpNestedLoops: - _, _, err := fixFieldRefIndexesForOperator(ctx, a, n, scope) + schema := thisNode.Schema() + expressions := thisNode.Expressions() + fixed, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, expressions...) if err != nil { return nil, true, err } - return n, true, nil - default: - return n, true, nil - } - }) -} + newNode, err := thisNode.WithUpdatedExpressions(fixed...) + if err != nil { + return nil, true, err + } + return newNode, same, nil -func fixGroupByFieldRefs(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { - return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) { - switch n := node.(type) { case *PlanOpGroupBy: - _, _, err := fixFieldRefIndexesForOperator(ctx, a, n, scope) + schema := thisNode.ChildOp.Schema() + aggregateExpressions := thisNode.Aggregates + fixedAggregateExpressions, aggregateSame, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, aggregateExpressions...) if err != nil { return nil, true, err } - return n, true, nil + + groupByExpressions := thisNode.GroupByExprs + fixedGroupByExpressions, groupBySame, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, groupByExpressions...) + if err != nil { + return nil, true, err + } + + newNode := NewPlanOpGroupBy(fixedAggregateExpressions, fixedGroupByExpressions, thisNode.ChildOp) + newNode.warnings = append(newNode.warnings, thisNode.warnings...) + return newNode, aggregateSame && groupBySame, nil + default: - return n, true, nil + return node, true, nil } }) } @@ -1015,24 +1047,35 @@ func getNestedLoopOperators(ctx context.Context, a *ExecutionPlanner, n types.Pl func fixFieldRefIndexes(ctx context.Context, scope *OptimizerScope, a *ExecutionPlanner, schema types.Schema, exp types.PlanExpression) (types.PlanExpression, bool, error) { return TransformExpr(exp, func(e types.PlanExpression) (types.PlanExpression, bool, error) { - switch e := e.(type) { + switch typedExpr := e.(type) { case *qualifiedRefPlanExpression: for i, col := range schema { newIndex := i - if e.Name() == col.ColumnName && e.tableName == col.RelationName { - if newIndex != e.columnIndex { - // update the column index - return newQualifiedRefPlanExpression(e.tableName, e.columnName, newIndex, e.dataType), false, nil + if strings.EqualFold(typedExpr.Name(), col.ColumnName) { + if len(typedExpr.tableName) > 0 { // do we have a qualifier? + if typedExpr.tableName == col.RelationName || typedExpr.tableName == col.AliasName { + if newIndex != typedExpr.columnIndex { + // update the column index + return newQualifiedRefPlanExpression(typedExpr.tableName, typedExpr.columnName, newIndex, typedExpr.dataType), false, nil + } + return e, true, nil + } + } else { // no qualifier + if newIndex != typedExpr.columnIndex { + // update the column index + return newQualifiedRefPlanExpression(typedExpr.tableName, typedExpr.columnName, newIndex, typedExpr.dataType), false, nil + } + return e, true, nil } - return e, true, nil } } - return nil, true, sql3.NewErrColumnNotFound(0, 0, e.Name()) + return nil, true, sql3.NewErrColumnNotFound(0, 0, typedExpr.Name()) } return e, true, nil }) } +// for a list of expressions and an operator schema, fix the references for any qualifiedRef expressions func fixFieldRefIndexesOnExpressions(ctx context.Context, scope *OptimizerScope, a *ExecutionPlanner, schema types.Schema, expressions ...types.PlanExpression) ([]types.PlanExpression, bool, error) { var result []types.PlanExpression var res types.PlanExpression @@ -1057,56 +1100,3 @@ func fixFieldRefIndexesOnExpressions(ctx context.Context, scope *OptimizerScope, } return expressions, true, nil } - -func fixFieldRefIndexesForOperator(ctx context.Context, a *ExecutionPlanner, node types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { - if _, ok := node.(types.ContainsExpressions); !ok { - return node, true, nil - } - - var schemas []types.Schema - for _, child := range node.Children() { - schemas = append(schemas, child.Schema()) - } - - if len(schemas) < 1 { - return node, true, nil - } - - n, sameC, err := TransformPlanOpExprsWithPlanOp(node, func(_ types.PlanOperator, e types.PlanExpression) (types.PlanExpression, bool, error) { - for _, schema := range schemas { - fixed, same, err := fixFieldRefIndexes(ctx, scope, a, schema, e) - if err == nil { - return fixed, same, nil - } - - if errors.Is(err, sql3.ErrColumnNotFound) { - continue - } - return nil, true, err - } - - return e, true, nil - }) - - if err != nil { - return nil, true, err - } - - sameJ := true - var cond types.PlanExpression - switch j := n.(type) { - case *PlanOpNestedLoops: - cond, sameJ, err = fixFieldRefIndexes(ctx, scope, a, j.Schema(), j.cond) - if err != nil { - return nil, true, err - } - if !sameJ { - n, err = j.WithUpdatedExpressions(cond) - if err != nil { - return nil, true, err - } - } - } - - return n, sameC && sameJ, nil -} diff --git a/sql3/planner/planwalker.go b/sql3/planner/planwalker.go index 2af498888..554021595 100644 --- a/sql3/planner/planwalker.go +++ b/sql3/planner/planwalker.go @@ -156,12 +156,18 @@ func InspectExpressionsWithPlanOp(op types.PlanOperator, f exprWithNodeInspector // If there was a transformation, the bool will be true, and an error if there was an error type PlanOpTransformFunc func(op types.PlanOperator) (types.PlanOperator, bool, error) -// TransformPlanOp applies a transformation function to the given plan op graph +// TransformPlanOp applies a depth first transformation function to the given plan op +// It returns a tuple that is the result of the transformation; the new PlanOperator, a bool that +// is true if the resultant PlanOperator has not been transformed or an error. +// If the TransformPlanOp has children it will iterate the children and call the transformation +// function on each of them in turn. If those operators are transformed, it will create a new operator +// with those children. The last step is to call the transformation on the passed PlanOperator func TransformPlanOp(op types.PlanOperator, f PlanOpTransformFunc) (types.PlanOperator, bool, error) { + thisOperator := op - children := op.Children() + children := thisOperator.Children() if len(children) == 0 { - return f(op) + return f(thisOperator) } var newChildren []types.PlanOperator @@ -185,17 +191,17 @@ func TransformPlanOp(op types.PlanOperator, f PlanOpTransformFunc) (types.PlanOp sameChildren := true if len(newChildren) > 0 { sameChildren = false - op, err = op.WithChildren(newChildren...) + thisOperator, err = thisOperator.WithChildren(newChildren...) if err != nil { return nil, true, err } } - op, sameOperator, err := f(op) + resultOperator, sameOperator, err := f(thisOperator) if err != nil { return nil, true, err } - return op, sameChildren && sameOperator, nil + return resultOperator, sameChildren && sameOperator, nil } // ParentContext is a struct that enables transformation functions to include a parent operator @@ -209,6 +215,7 @@ type ParentContextFunc func(c ParentContext) (types.PlanOperator, bool, error) type ParentSelectorFunc func(c ParentContext) bool +// TransformPlanOpWithParent applies a transformation function to a plan operator in the context that plan operators parent func TransformPlanOpWithParent(op types.PlanOperator, s ParentSelectorFunc, f ParentContextFunc) (types.PlanOperator, bool, error) { return planOpWithParentHelper(ParentContext{op, nil, -1}, s, f) } @@ -252,11 +259,11 @@ func planOpWithParentHelper(c ParentContext, s ParentSelectorFunc, f ParentConte } } - operator, sameOperator, err := f(ParentContext{operator, c.Parent, c.ChildCount}) + resultOperator, sameOperator, err := f(ParentContext{operator, c.Parent, c.ChildCount}) if err != nil { return nil, true, err } - return operator, sameChildren && sameOperator, nil + return resultOperator, sameChildren && sameOperator, nil } // ExprWithPlanOpFunc is a function that given an expression and the node @@ -362,11 +369,13 @@ func TransformSinglePlanOpExpressions(op types.PlanOperator, f ExprFunc) (types. return op, true, nil } -// TransformExpr applies a transformation function to an expression +// TransformExpr applies a depth first transformation function to an expression func TransformExpr(expr types.PlanExpression, f ExprFunc) (types.PlanExpression, bool, error) { + thisExpr := expr + children := expr.Children() if len(children) == 0 { - return f(expr) + return f(thisExpr) } var ( @@ -392,22 +401,24 @@ func TransformExpr(expr types.PlanExpression, f ExprFunc) (types.PlanExpression, sameChildren := true if len(newChildren) > 0 { sameChildren = false - expr, err = expr.WithChildren(newChildren...) + thisExpr, err = thisExpr.WithChildren(newChildren...) if err != nil { return nil, true, err } } - expr, sameExpr, err := f(expr) + resultExpr, sameExpr, err := f(thisExpr) if err != nil { return nil, true, err } - return expr, sameChildren && sameExpr, nil + return resultExpr, sameChildren && sameExpr, nil } -// TransformExprWithPlanOp applies a transformation function to an expression in the context of a plan operator +// TransformExprWithPlanOp applies a depth first transformation function to an expression in the context of a plan operator func TransformExprWithPlanOp(n types.PlanOperator, e types.PlanExpression, f ExprWithPlanOpFunc) (types.PlanExpression, bool, error) { - children := e.Children() + thisExpr := e + + children := thisExpr.Children() if len(children) == 0 { return f(n, e) } @@ -435,15 +446,15 @@ func TransformExprWithPlanOp(n types.PlanOperator, e types.PlanExpression, f Exp sameChilren := true if len(newChildren) > 0 { sameChilren = false - e, err = e.WithChildren(newChildren...) + thisExpr, err = thisExpr.WithChildren(newChildren...) if err != nil { return nil, true, err } } - e, sameExpr, err := f(n, e) + resultExpr, sameExpr, err := f(n, thisExpr) if err != nil { return nil, true, err } - return e, sameChilren && sameExpr, nil + return resultExpr, sameChilren && sameExpr, nil } diff --git a/sql3/planner/types/planexpression.go b/sql3/planner/types/planexpression.go index 2d537842f..2a49677d7 100644 --- a/sql3/planner/types/planexpression.go +++ b/sql3/planner/types/planexpression.go @@ -2,6 +2,7 @@ package types import ( "context" + "fmt" "github.com/featurebasedb/featurebase/v3/sql3/parser" ) @@ -24,6 +25,8 @@ const ( // PlanExpression is an expression node for an execution plan type PlanExpression interface { + fmt.Stringer + // evaluates expression based on current row Evaluate(currentRow []interface{}) (interface{}, error) @@ -50,6 +53,8 @@ type AggregationBuffer interface { // Interface to an expression that is a an aggregate type Aggregable interface { + fmt.Stringer + NewBuffer() (AggregationBuffer, error) AggType() AggregateFunctionType AggExpression() PlanExpression diff --git a/sql3/sql_complex_test.go b/sql3/sql_complex_test.go index 96f8d9c10..b6d8156a3 100644 --- a/sql3/sql_complex_test.go +++ b/sql3/sql_complex_test.go @@ -19,6 +19,23 @@ import ( "github.com/stretchr/testify/assert" ) +func TestPlanner_Misc(t *testing.T) { + + d, err := parser.StringToDecimal("12.345678") + if err != nil { + t.Fatal(err) + } + + assert.True(t, d.EqualTo(pql.NewDecimal(12345678, 6))) + + d = parser.FloatToDecimalWithScale(12.345678, 6) + if err != nil { + t.Fatal(err) + } + + assert.True(t, d.EqualTo(pql.NewDecimal(12345678, 6))) +} + func TestPlanner_Show(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() diff --git a/sql3/test/defs/defs_join.go b/sql3/test/defs/defs_join.go index 80a5232d1..e99586f67 100644 --- a/sql3/test/defs/defs_join.go +++ b/sql3/test/defs/defs_join.go @@ -1,5 +1,7 @@ package defs +import "github.com/molecula/featurebase/v3/pql" + // join tests var joinTestsUsers = TableTest{ name: "jointestusers", @@ -54,10 +56,49 @@ var joinTests = TableTest{ hdr("", fldTypeDecimal2), ), ExpRows: rows( - row(int64(1), float64(22.98)), - row(int64(0), float64(3.99)), - row(int64(2), float64(16.98)), - row(int64(3), float64(5.99)), + row(int64(1), pql.NewDecimal(2298, 2)), + row(int64(0), pql.NewDecimal(399, 2)), + row(int64(2), pql.NewDecimal(1698, 2)), + row(int64(3), pql.NewDecimal(599, 2)), + ), + Compare: CompareExactOrdered, + }, + { + name: "innerjoin-aggregate-groupby-sum-filter", + SQLs: sqls( + "select sum(price) from orders o inner join users u on o.userid = u._id where u.age > 20;", + ), + ExpHdrs: hdrs( + hdr("", fldTypeDecimal2), + ), + ExpRows: rows( + row(pql.NewDecimal(2696, 2)), + ), + Compare: CompareExactOrdered, + }, + { + name: "innerjoin-aggregate-groupby-count-distinct-filter", + SQLs: sqls( + "SELECT COUNT(DISTINCT u.name) FROM orders o JOIN users u ON o.userid = u._id WHERE o.price > 10;", + ), + ExpHdrs: hdrs( + hdr("", fldTypeInt), + ), + ExpRows: rows( + row(int64(2)), + ), + Compare: CompareExactOrdered, + }, + { + name: "innerjoin-aggregate-groupby-count-filter", + SQLs: sqls( + "SELECT COUNT(u.name) FROM orders o JOIN users u ON o.userid = u._id WHERE o.price > 10;", + ), + ExpHdrs: hdrs( + hdr("", fldTypeInt), + ), + ExpRows: rows( + row(int64(2)), ), Compare: CompareExactOrdered, }, diff --git a/sql3/test/defs/defs_unops.go b/sql3/test/defs/defs_unops.go index ca3fd7e01..9cbb611ad 100644 --- a/sql3/test/defs/defs_unops.go +++ b/sql3/test/defs/defs_unops.go @@ -1,6 +1,10 @@ package defs -import "time" +import ( + "time" + + "github.com/molecula/featurebase/v3/pql" +) var unaryOpExprWithInt = TableTest{ Table: tbl( @@ -157,7 +161,7 @@ var unaryOpExprWithDecimal = TableTest{ hdr("", fldTypeDecimal2), ), ExpRows: rows( - row(float64(-12.34)), + row(pql.NewDecimal(-1234, 2)), ), Compare: CompareExactUnordered, }, @@ -175,7 +179,7 @@ var unaryOpExprWithDecimal = TableTest{ hdr("", fldTypeDecimal2), ), ExpRows: rows( - row(float64(12.34)), + row(pql.NewDecimal(1234, 2)), ), Compare: CompareExactUnordered, }, diff --git a/sql3/test/helpers.go b/sql3/test/helpers.go index 237fc837a..db4f91c41 100644 --- a/sql3/test/helpers.go +++ b/sql3/test/helpers.go @@ -20,6 +20,9 @@ func MustQueryRows(tb testing.TB, svr *pilosa.Server, q string) ([][]interface{} return nil, nil, err } + // get the plan so that code runs during testing + _ = stmt.Plan() + ocolumns := stmt.Schema() rowIter, err := stmt.Iterator(ctx, nil)