scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fixed join bugs by fixing query optimizer (fb-1699, fb-1700) (#2307)

Browse source 
this commit changes the way the plan is retrieved; implements Stringer on types.PlanExpression in preparation for HAVING support; removes last vestiges internal float64 arithmetic; implements a filter on PlanOpFilter; fixes various bugs in the PlanOptimizer when rewriting qualified references

* fixed selects with unqualified identifiers

* handle bad and non-existent query param inputs more appropriately

* added test coverage for PlanExpression Stringer

Co-authored-by: Matthew Jaffee <jaffee@pilosa.com>
(cherry picked from commit 5f662d2bce)
This commit is contained in:
pokeeffe-molecula 2022-11-21 14:31:45 -06:00 • committed by Fletcher Haynes
parent cd260fe665
commit b17ee6a203
24 changed files with 664 additions and 202 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
http_handler.go
View file

@ -1398,6 +1398,17 @@ func (h *Handler) writeBadRequest(w http.ResponseWriter, r *http.Request, err er
// handlePostSQL handles /sql requests.
func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
includePlan := false
includePlanValue := r.URL.Query().Get("plan")
if len(includePlanValue) > 0 {
var err error
includePlan, err = strconv.ParseBool(includePlanValue)
if err != nil {
h.writeBadRequest(w, r, err)
return
}
}
b, err := io.ReadAll(r.Body)
@ -1466,6 +1477,17 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
}
}
writePlan := func(plan map[string]interface{}) {
if plan != nil && includePlan {
planBytes, err := json.Marshal(plan)
if err != nil {
planBytes = []byte(`"PROBLEM ENCODING QUERY PLAN"`)
}
w.Write([]byte(`,"queryPlan":`))
w.Write(planBytes)
}
}
// Get a query iterator.
iter, err := rootOperator.Iterator(r.Context(), nil)
if err != nil {
@ -1529,6 +1551,7 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
writeError(rowErr)
writeWarnings(rootOperator.Warnings())
writePlan(rootOperator.Plan())
}
type SQLField struct {

5
sql3/parser/astdatatype.go
View file

@ -257,6 +257,11 @@ func FloatToDecimal(v float64) pql.Decimal {
return pql.NewDecimal(unscaledValue, int64(scale))
}
func FloatToDecimalWithScale(v float64, s int64) pql.Decimal {
unscaledValue := int64(v * math.Pow(10, float64(s)))
return pql.NewDecimal(unscaledValue, int64(s))
}
func NumDecimalPlaces(v string) int {
i := strings.IndexByte(v, '.')
if i > -1 {

2
sql3/planner/executionplanner.go
View file

@ -79,12 +79,10 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen
default:
return nil, sql3.NewErrInternalf("cannot plan statement: %T", stmt)
}
// Optimize the plan.
if err == nil {
rootOperator, err = p.optimizePlan(ctx, rootOperator)
}
return rootOperator, err
}

208
sql3/planner/expression.go
View file

@ -166,6 +166,10 @@ func (n *unaryOpPlanExpression) Type() parser.ExprDataType {
return n.resultDataType
}
func (n *unaryOpPlanExpression) String() string {
return fmt.Sprintf("%s%s", n.op.String(), n.rhs.String())
}
func (n *unaryOpPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -233,7 +237,11 @@ func (n *unaryOpPlanExpression) plusWithTypeCheck(rhs interface{}) (interface{},
case *parser.DataTypeDecimal:
nr, nrok := rhs.(pql.Decimal)
if nrok {
return +(nr.Float64()), nil
val := nr.Value()
if !val.IsInt64() {
return nil, sql3.NewErrInternalf("decimal value overflow: %v", rhs)
}
return pql.NewDecimal(+val.Int64(), nr.Scale), nil
}
return nil, sql3.NewErrInternalf("unexpected incompatible types '%T", rhs)
@ -266,7 +274,12 @@ func (n *unaryOpPlanExpression) minusWithTypeCheck(rhs interface{}) (interface{}
case *parser.DataTypeDecimal:
nr, nrok := rhs.(pql.Decimal)
if nrok {
return -(nr.Float64()), nil
val := nr.Value()
if !val.IsInt64() {
return nil, sql3.NewErrInternalf("decimal value overflow: %v", rhs)
}
return pql.NewDecimal(-val.Int64(), nr.Scale), nil
}
return nil, sql3.NewErrInternalf("unexpected incompatible types '%T", rhs)
@ -644,6 +657,10 @@ func (n *binOpPlanExpression) Type() parser.ExprDataType {
return n.resultDataType
}
func (n *binOpPlanExpression) String() string {
return fmt.Sprintf("%s%s%s", n.lhs.String(), n.op.String(), n.rhs.String())
}
func (n *binOpPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -711,6 +728,10 @@ func (n *rangePlanExpression) Type() parser.ExprDataType {
return n.resultDataType
}
func (n *rangePlanExpression) String() string {
return fmt.Sprintf("between %s and %s", n.lhs.String(), n.rhs.String())
}
func (n *rangePlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -900,6 +921,24 @@ func (n *casePlanExpression) Type() parser.ExprDataType {
return n.resultDataType
}
func (n *casePlanExpression) String() string {
var result string
if n.baseExpr != nil {
result = fmt.Sprintf("case %s", n.baseExpr)
} else {
result = "case"
}
for _, blk := range n.blocks {
result += fmt.Sprintf(" %s", blk.String())
}
if n.elseExpr != nil {
result += fmt.Sprintf(" else %s end", n.elseExpr)
} else {
result += " end"
}
return result
}
func (n *casePlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -919,11 +958,45 @@ func (n *casePlanExpression) Plan() map[string]interface{} {
}
func (n *casePlanExpression) Children() []types.PlanExpression {
return []types.PlanExpression{}
result := make([]types.PlanExpression, 0)
if n.baseExpr != nil {
result = append(result, n.baseExpr)
}
result = append(result, n.blocks...)
if n.elseExpr != nil {
result = append(result, n.elseExpr)
}
return result
}
func (n *casePlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
return n, nil
currentLen := 0
if n.baseExpr != nil {
currentLen += 1
}
currentLen += len(n.blocks)
if n.elseExpr != nil {
currentLen += 1
}
if len(children) != currentLen {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
offset := 0
var newBaseExpr types.PlanExpression
if n.baseExpr != nil {
newBaseExpr = children[offset]
offset += 1
}
newBlocks := make([]types.PlanExpression, len(n.blocks))
copy(newBlocks[0:], children[offset:offset+len(n.blocks)])
offset += len(n.blocks)
var newElseExpr types.PlanExpression
if n.elseExpr != nil {
newElseExpr = children[offset]
}
return newCasePlanExpression(newBaseExpr, newBlocks, newElseExpr, n.resultDataType), nil
}
// caseBlockPlanExpression is for case blocks
@ -947,6 +1020,10 @@ func (n *caseBlockPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeBool()
}
func (n *caseBlockPlanExpression) String() string {
return fmt.Sprintf("when %s then %s end", n.condition.String(), n.body.String())
}
func (n *caseBlockPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -964,7 +1041,10 @@ func (n *caseBlockPlanExpression) Children() []types.PlanExpression {
}
func (n *caseBlockPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
return n, nil
if len(children) != 2 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newCaseBlockPlanExpression(children[0], children[1]), nil
}
// subqueryPlanExpression is a select statement (when used in an expression)
@ -1009,6 +1089,10 @@ func (n *subqueryPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeBool()
}
func (n *subqueryPlanExpression) String() string {
return n.op.String()
}
func (n *subqueryPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1110,6 +1194,13 @@ func (n *betweenOpPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeBool()
}
func (n *betweenOpPlanExpression) String() string {
if n.op == parser.BETWEEN {
return fmt.Sprintf("between %s and %s", n.lhs.String(), n.rhs.String())
}
return fmt.Sprintf("not between %s and %s", n.lhs.String(), n.rhs.String())
}
func (n *betweenOpPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1127,7 +1218,7 @@ func (n *betweenOpPlanExpression) Children() []types.PlanExpression {
}
func (n *betweenOpPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
if len(children) != 1 {
if len(children) != 2 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newBetweenOpPlanExpression(children[0], n.op, children[1]), nil
@ -1317,6 +1408,17 @@ func (n *inOpPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeBool()
}
func (n *inOpPlanExpression) String() string {
s := n.lhs.String()
if n.op == parser.NOTIN {
s += " not "
}
s += " in ("
s += n.rhs.String()
s += ")"
return s
}
func (n *inOpPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1335,7 +1437,7 @@ func (n *inOpPlanExpression) Children() []types.PlanExpression {
}
func (n *inOpPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
if len(children) != 1 {
if len(children) != 2 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newInOpPlanExpression(children[0], n.op, children[1]), nil
@ -1375,6 +1477,17 @@ func (n *callPlanExpression) Type() parser.ExprDataType {
return n.dataType
}
func (n *callPlanExpression) String() string {
args := ""
for idx, arg := range n.args {
if idx > 0 {
args += ", "
}
args += arg.String()
}
return fmt.Sprintf("%s(%s)", n.name, args)
}
func (n *callPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1427,6 +1540,10 @@ func (n *aliasPlanExpression) Type() parser.ExprDataType {
return n.expr.Type()
}
func (n *aliasPlanExpression) String() string {
return fmt.Sprintf("%s as %s", n.expr.String(), n.aliasName)
}
func (n *aliasPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1513,6 +1630,13 @@ func (n *qualifiedRefPlanExpression) Type() parser.ExprDataType {
return n.dataType
}
func (n *qualifiedRefPlanExpression) String() string {
if len(n.tableName) > 0 {
return fmt.Sprintf("%s.%s", n.tableName, n.columnName)
}
return n.columnName
}
func (n *qualifiedRefPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1571,6 +1695,10 @@ func (n *variableRefPlanExpression) Type() parser.ExprDataType {
return n.dataType
}
func (n *variableRefPlanExpression) String() string {
return fmt.Sprintf("@%s", n.name)
}
func (n *variableRefPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1602,6 +1730,10 @@ func (n *nullLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeVoid()
}
func (n *nullLiteralPlanExpression) String() string {
return "null"
}
func (n *nullLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1636,6 +1768,10 @@ func (n *intLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeInt()
}
func (n *intLiteralPlanExpression) String() string {
return fmt.Sprintf("%d", n.value)
}
func (n *intLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1678,6 +1814,10 @@ func (n *floatLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeDecimal(int64(scale))
}
func (n *floatLiteralPlanExpression) String() string {
return n.value
}
func (n *floatLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1713,6 +1853,10 @@ func (n *boolLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeBool()
}
func (n *boolLiteralPlanExpression) String() string {
return fmt.Sprintf("%v", n.value)
}
func (n *boolLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1748,6 +1892,10 @@ func (n *dateLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeTimestamp()
}
func (n *dateLiteralPlanExpression) String() string {
return n.value.Format(time.RFC3339Nano)
}
func (n *dateLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1783,6 +1931,10 @@ func (n *stringLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeString()
}
func (n *stringLiteralPlanExpression) String() string {
return fmt.Sprintf("'%s'", n.value)
}
func (n *stringLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -2006,6 +2158,10 @@ func (n *castPlanExpression) Type() parser.ExprDataType {
return n.targetType
}
func (n *castPlanExpression) String() string {
return fmt.Sprintf("cast(%s as %s)", n.lhs.String(), n.targetType.TypeName())
}
func (n *castPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -2046,6 +2202,17 @@ func (n *exprListPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeVoid()
}
func (n *exprListPlanExpression) String() string {
var s string
for idx, expr := range n.exprs {
if idx > 0 {
s += ", "
}
s += expr.String()
}
return fmt.Sprintf("(%s)", s)
}
func (n *exprListPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -2062,7 +2229,10 @@ func (n *exprListPlanExpression) Children() []types.PlanExpression {
}
func (n *exprListPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
return n, nil
if len(children) != len(n.exprs) {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newExprListExpression(children), nil
}
// exprSetLiteralPlanExpression is a set literal
@ -2122,6 +2292,17 @@ func (n *exprSetLiteralPlanExpression) Type() parser.ExprDataType {
return n.dataType
}
func (n *exprSetLiteralPlanExpression) String() string {
var members string
for idx, m := range n.members {
if idx > 0 {
members += ", "
}
members += m.String()
}
return fmt.Sprintf("[%s]", members)
}
func (n *exprSetLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -2192,6 +2373,17 @@ func (n *exprTupleLiteralPlanExpression) Type() parser.ExprDataType {
return n.dataType
}
func (n *exprTupleLiteralPlanExpression) String() string {
members := ""
for idx, m := range n.members {
if idx > 0 {
members += ", "
}
members += m.String()
}
return fmt.Sprintf("{%s}", members)
}
func (n *exprTupleLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)

84
sql3/planner/expression_it_test.go Normal file
View file

@ -0,0 +1,84 @@
package planner
import (
"testing"
"time"
"github.com/molecula/featurebase/v3/sql3/parser"
"github.com/molecula/featurebase/v3/sql3/planner/types"
"github.com/stretchr/testify/assert"
)
func TestExpressions(t *testing.T) {
t.Run("StringerTest", func(t *testing.T) {
uop := newUnaryOpPlanExpression(parser.PLUS, newIntLiteralPlanExpression(10), parser.NewDataTypeInt())
assert.Equal(t, uop.String(), "+10")
bop := newBinOpPlanExpression(newIntLiteralPlanExpression(10), parser.PLUS, newIntLiteralPlanExpression(20), parser.NewDataTypeInt())
assert.Equal(t, bop.String(), "10+20")
rop := newRangeOpPlanExpression(newIntLiteralPlanExpression(10), newIntLiteralPlanExpression(20), parser.NewDataTypeInt())
assert.Equal(t, rop.String(), "between 10 and 20")
cop := newCasePlanExpression(newStringLiteralPlanExpression("foo"),
[]types.PlanExpression{
newCaseBlockPlanExpression(newStringLiteralPlanExpression("20"), newIntLiteralPlanExpression(20)),
newCaseBlockPlanExpression(newStringLiteralPlanExpression("30"), newIntLiteralPlanExpression(20)),
},
newIntLiteralPlanExpression(20), parser.NewDataTypeInt())
assert.Equal(t, cop.String(), "case 'foo' when '20' then 20 end when '30' then 20 end else 20 end")
bwop := newBetweenOpPlanExpression(newIntLiteralPlanExpression(10), parser.BETWEEN, newIntLiteralPlanExpression(20))
assert.Equal(t, bwop.String(), "between 10 and 20")
iop := newInOpPlanExpression(newIntLiteralPlanExpression(10), parser.IN, newIntLiteralPlanExpression(20))
assert.Equal(t, iop.String(), "10 in (20)")
callop := newCallPlanExpression("foo", []types.PlanExpression{newIntLiteralPlanExpression(10)}, parser.NewDataTypeInt())
assert.Equal(t, callop.String(), "foo(10)")
alop := newAliasPlanExpression("frobny", newIntLiteralPlanExpression(10))
assert.Equal(t, alop.String(), "10 as frobny")
qrop := newQualifiedRefPlanExpression("foo", "bar", 1, parser.NewDataTypeInt())
assert.Equal(t, qrop.String(), "foo.bar")
vop := newVariableRefPlanExpression("foo", 1, parser.NewDataTypeInt())
assert.Equal(t, vop.String(), "@foo")
nulop := newNullLiteralPlanExpression()
assert.Equal(t, nulop.String(), "null")
ilop := newIntLiteralPlanExpression(10)
assert.Equal(t, ilop.String(), "10")
flop := newFloatLiteralPlanExpression("12.3456")
assert.Equal(t, flop.String(), "12.3456")
blop := newBoolLiteralPlanExpression(false)
assert.Equal(t, blop.String(), "false")
tm, _ := time.ParseInLocation(time.RFC3339, "2012-11-01T22:08:41+00:00", time.UTC)
dlop := newDateLiteralPlanExpression(tm)
assert.Equal(t, dlop.String(), "2012-11-01T22:08:41Z")
slop := newStringLiteralPlanExpression("foo")
assert.Equal(t, slop.String(), "'foo'")
ctop := newCastPlanExpression(newIntLiteralPlanExpression(10), parser.NewDataTypeString())
assert.Equal(t, ctop.String(), "cast(10 as STRING)")
elop := newExprListExpression([]types.PlanExpression{newStringLiteralPlanExpression("foo"), newStringLiteralPlanExpression("bar")})
assert.Equal(t, elop.String(), "('foo', 'bar')")
stlop := newExprSetLiteralPlanExpression([]types.PlanExpression{newStringLiteralPlanExpression("foo"), newStringLiteralPlanExpression("bar")}, parser.NewDataTypeString())
assert.Equal(t, stlop.String(), "['foo', 'bar']")
tplop := newExprTupleLiteralPlanExpression([]types.PlanExpression{newStringLiteralPlanExpression("foo"), newStringLiteralPlanExpression("bar")}, parser.NewDataTypeString())
assert.Equal(t, tplop.String(), "{'foo', 'bar'}")
})
}

64
sql3/planner/expressionagg.go
View file

@ -120,6 +120,10 @@ func (n *countPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *countPlanExpression) String() string {
return fmt.Sprintf("count(%s)", n.arg.String())
}
func (n *countPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -179,6 +183,10 @@ func (n *countDistinctPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *countDistinctPlanExpression) String() string {
return fmt.Sprintf("count(distinct %s)", n.arg.String())
}
func (n *countDistinctPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -197,13 +205,12 @@ func (n *countDistinctPlanExpression) WithChildren(children ...types.PlanExpress
// aggregator for the SUM function
type aggregateSum struct {
sum float64
sum interface{}
expr types.PlanExpression
}
func NewAggSumBuffer(child types.PlanExpression) *aggregateSum {
return &aggregateSum{
sum: float64(0),
expr: child,
}
}
@ -230,7 +237,17 @@ func (m *aggregateSum) Update(ctx context.Context, row types.Row) error {
if !ok {
return sql3.NewErrInternalf("unexpected type conversion '%T'", v)
}
m.sum += val.Float64()
var dsum pql.Decimal
if m.sum != nil {
dsum, ok = m.sum.(pql.Decimal)
if !ok {
return sql3.NewErrInternalf("unexpected type conversion '%T'", m.sum)
}
} else {
dsum = pql.NewDecimal(0, dataType.Scale)
}
dsum = pql.AddDecimal(dsum, val)
m.sum = dsum
default:
return sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", dataType)
}
@ -238,7 +255,17 @@ func (m *aggregateSum) Update(ctx context.Context, row types.Row) error {
}
func (m *aggregateSum) Eval(ctx context.Context) (interface{}, error) {
return m.sum, nil
switch m.expr.Type().(type) {
case *parser.DataTypeDecimal:
dsum, ok := m.sum.(pql.Decimal)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type conversion '%T'", m.sum)
}
return dsum, nil
default:
return nil, sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", m.expr.Type())
}
}
// sumPlanExpression handles SUM()
@ -284,6 +311,10 @@ func (n *sumPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *sumPlanExpression) String() string {
return fmt.Sprintf("sum(%s)", n.arg.String())
}
func (n *sumPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -293,11 +324,16 @@ func (n *sumPlanExpression) Plan() map[string]interface{} {
}
func (n *sumPlanExpression) Children() []types.PlanExpression {
return nil
return []types.PlanExpression{
n.arg,
}
}
func (n *sumPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
return n, nil
if len(children) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newSumPlanExpression(children[0], n.returnDataType), nil
}
// aggregator for AVG
@ -387,6 +423,10 @@ func (n *avgPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *avgPlanExpression) String() string {
return fmt.Sprintf("avg(%s)", n.arg.String())
}
func (n *avgPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -480,6 +520,10 @@ func (n *minPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *minPlanExpression) String() string {
return fmt.Sprintf("min(%s)", n.arg.String())
}
func (n *minPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -573,6 +617,10 @@ func (n *maxPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *maxPlanExpression) String() string {
return fmt.Sprintf("max(%s)", n.arg.String())
}
func (n *maxPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -636,6 +684,10 @@ func (n *percentilePlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *percentilePlanExpression) String() string {
return fmt.Sprintf("percentile(%s)", n.arg.String())
}
func (n *percentilePlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)

2
sql3/planner/expressionanalyzer.go
View file

@ -337,7 +337,7 @@ func (p *ExecutionPlanner) analyzeUnaryExpression(expr *parser.UnaryExpr, scope
}
if typeIsInteger(x.DataType()) {
expr.ResultDataType = parser.NewDataTypeInt()
} else if typeIsFloat(x.DataType()) {
} else if typeIsDecimal(x.DataType()) {
fd, ok := x.DataType().(*parser.DataTypeDecimal)
if !ok {
return nil, sql3.NewErrInternalf("unexpected data type")

8
sql3/planner/expressionanalyzercall.go
View file

@ -67,7 +67,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser
}
//make sure the ref is sum-able
if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType())) {
if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType())) {
return nil, sql3.NewErrIntOrDecimalExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column)
}
@ -95,7 +95,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser
}
//make sure the ref is avg-able
if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType())) {
if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType())) {
return nil, sql3.NewErrIntOrDecimalExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column)
}
@ -123,7 +123,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser
}
//make sure the ref is percentilable-able
if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType()) || typeIsTimestamp(ref.DataType())) {
if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType()) || typeIsTimestamp(ref.DataType())) {
return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column)
}
@ -164,7 +164,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser
}
// make sure the ref is min/max-able
if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType()) || typeIsTimestamp(ref.DataType())) {
if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType()) || typeIsTimestamp(ref.DataType())) {
return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column)
}

4
sql3/planner/expressiontypes.go
View file

@ -481,8 +481,8 @@ func typeIsTimestamp(testType parser.ExprDataType) bool {
}
}
// returns true if the type is a float
func typeIsFloat(testType parser.ExprDataType) bool {
// returns true if the type is a decimal
func typeIsDecimal(testType parser.ExprDataType) bool {
switch testType.(type) {
case *parser.DataTypeDecimal:
return true

34
sql3/planner/opfilter.go
View file

@ -61,6 +61,7 @@ func (p *PlanOpFilter) Plan() map[string]interface{} {
ps = append(ps, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeName()))
}
result["_schema"] = ps
result["predicate"] = p.Predicate.Plan()
result["child"] = p.ChildOp.Plan()
return result
}
@ -77,6 +78,23 @@ func (p *PlanOpFilter) Warnings() []string {
return p.warnings
}
func (p *PlanOpFilter) Expressions() []types.PlanExpression {
if p.Predicate != nil {
return []types.PlanExpression{
p.Predicate,
}
}
return []types.PlanExpression{}
}
func (p *PlanOpFilter) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) {
if len(exprs) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs))
}
p.Predicate = exprs[0]
return p, nil
}
type filterIterator struct {
predicate types.PlanExpression
child types.RowIterator
@ -92,6 +110,18 @@ func newFilterIterator(ctx context.Context, predicate types.PlanExpression, chil
}
func (i *filterIterator) Next(ctx context.Context) (types.Row, error) {
//TODO (pok) - actually implement the filter
return i.child.Next(ctx)
for {
row, err := i.child.Next(ctx)
if err != nil {
return nil, err
}
matches, err := conditionIsTrue(ctx, row, i.predicate)
if err != nil {
return nil, err
}
if !matches {
continue
}
return row, nil
}
}

7
sql3/planner/opgroupby.go
View file

@ -49,7 +49,7 @@ func (p *PlanOpGroupBy) Schema() types.Schema {
offset := len(p.GroupByExprs)
for idx, agg := range p.Aggregates {
s := &types.PlannerColumn{
ColumnName: "",
ColumnName: agg.String(),
RelationName: "",
Type: agg.Type(),
}
@ -86,7 +86,7 @@ func (p *PlanOpGroupBy) WithChildren(children ...types.PlanOperator) (types.Plan
func (p *PlanOpGroupBy) Expressions() []types.PlanExpression {
result := []types.PlanExpression{}
result = append(result, p.GroupByExprs...)
result = append(result, p.Aggregates...)
return result
}
@ -94,8 +94,7 @@ func (p *PlanOpGroupBy) WithUpdatedExpressions(exprs ...types.PlanExpression) (t
if len(exprs) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs))
}
p.GroupByExprs = exprs
return p, nil
return NewPlanOpGroupBy(exprs, p.GroupByExprs, p.ChildOp), nil
}
func (p *PlanOpGroupBy) Plan() map[string]interface{} {

17
sql3/planner/opnestedloops.go
View file

@ -38,6 +38,7 @@ func (p *PlanOpNestedLoops) Plan() map[string]interface{} {
result["_schema"] = ps
result["top"] = p.top.Plan()
result["bottom"] = p.bottom.Plan()
result["condition"] = p.cond.Plan()
return result
}
@ -124,8 +125,6 @@ type nestedLoopsIter struct {
rowSize int
originalRow types.Row
bottomRows RowCache
}
func newNestedLoopsIter(ctx context.Context, jt joinType, top types.RowIterator, bottom types.RowIterable, scopeRow types.Row, joinCondition types.PlanExpression, rowWidth int, originalRow types.Row) *nestedLoopsIter {
@ -136,7 +135,6 @@ func newNestedLoopsIter(ctx context.Context, jt joinType, top types.RowIterator,
cond: joinCondition,
rowSize: rowWidth,
originalRow: originalRow,
bottomRows: newInMemoryRowCache(),
ctx: ctx,
}
}
@ -153,11 +151,14 @@ func (i *nestedLoopsIter) loadTop(ctx context.Context) error {
i.topRow = i.originalRow.Append(r)
i.foundMatch = false
//DEBUG log.Printf("top row %v", i.topRow)
return nil
}
func (i *nestedLoopsIter) loadBottom(ctx context.Context) (row types.Row, err error) {
if i.bottom == nil {
// DEBUG log.Printf("bottom row initializing iterator...")
var iter types.RowIterator
iter, err = i.bottomProvider.Iterator(ctx, i.topRow)
if err != nil {
@ -169,16 +170,15 @@ func (i *nestedLoopsIter) loadBottom(ctx context.Context) (row types.Row, err er
rightRow, err := i.bottom.Next(ctx)
if err != nil {
if err == types.ErrNoMoreRows {
// DEBUG log.Printf("bottom end of rows")
i.bottom = nil
i.topRow = nil
return nil, types.ErrNoMoreRows
}
return nil, err
}
err = i.bottomRows.Add(rightRow)
if err != nil {
return nil, err
}
//DEBUG log.Printf("bottom row %v", rightRow)
return rightRow, nil
}
@ -247,7 +247,8 @@ func (i *nestedLoopsIter) Next(ctx context.Context) (types.Row, error) {
i.foundMatch = true
//DEBUG log.Printf("Join result %v", row)
// DEBUG log.Printf("join result %v", row)
return row, nil
}
}

36
sql3/planner/oppqltablescan.go
View file

@ -85,12 +85,17 @@ func (p *PlanOpPQLTableScan) Schema() types.Schema {
return result
}
for _, col := range table.Fields {
result = append(result, &types.PlannerColumn{
ColumnName: col.Name,
RelationName: p.tableName,
Type: fieldSQLDataType(col),
})
for _, col := range p.columns {
for _, fld := range table.Fields {
if strings.EqualFold(fld.Name, col) {
result = append(result, &types.PlannerColumn{
ColumnName: fld.Name,
RelationName: p.tableName,
Type: fieldSQLDataType(fld),
})
break
}
}
}
return result
}
@ -149,15 +154,20 @@ func (i *tableScanRowIter) Next(ctx context.Context) (types.Row, error) {
}
return nil, err
}
i.rowWidth = len(table.Fields)
i.rowWidth = len(i.columns)
i.columnMap = make(map[string]*targetColumn)
for idx, fld := range table.Fields {
i.columnMap[fld.Name] = &targetColumn{
columnIdx: idx,
srcColumnIdx: -1,
columnName: fld.Name,
dataType: fieldSQLDataType(fld),
for idx, col := range i.columns {
for _, fld := range table.Fields {
if strings.EqualFold(col, fld.Name) {
i.columnMap[fld.Name] = &targetColumn{
columnIdx: idx,
srcColumnIdx: -1,
columnName: fld.Name,
dataType: fieldSQLDataType(fld),
}
break
}
}
}

7
sql3/planner/opprojection.go
View file

@ -60,12 +60,12 @@ func (p *PlanOpProjection) WithChildren(children ...types.PlanOperator) (types.P
func (p *PlanOpProjection) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_op"] = fmt.Sprintf("%T", p)
result["__op"] = fmt.Sprintf("%T", p)
sc := make([]string, 0)
for _, e := range p.Schema() {
sc = append(sc, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeName()))
}
result["_schema"] = sc
result["__schema"] = sc
result["child"] = p.ChildOp.Plan()
@ -73,7 +73,7 @@ func (p *PlanOpProjection) Plan() map[string]interface{} {
for _, e := range p.Projections {
ps = append(ps, e.Plan())
}
result["projections"] = ps
result["_projections"] = ps
return result
}
@ -100,7 +100,6 @@ func ExpressionToColumn(e types.PlanExpression) *types.PlannerColumn {
if n, ok := e.(types.IdentifiableByName); ok {
name = n.Name()
} else {
//TODO(pok) - implement this
name = "" //e.String()
}

1
sql3/planner/opquery.go
View file

@ -28,6 +28,7 @@ func NewPlanOpQuery(child types.PlanOperator, sql string) *PlanOpQuery {
return &PlanOpQuery{
ChildOp: child,
warnings: make([]string, 0),
sql: sql,
}
}

2
sql3/planner/oprelalias.go
View file

@ -28,7 +28,7 @@ func NewPlanOpRelAlias(alias string, child types.PlanOperator) *PlanOpRelAlias {
func (p *PlanOpRelAlias) Schema() types.Schema {
schema := p.ChildOp.Schema()
for _, s := range schema {
s.RelationName = p.alias
s.AliasName = p.alias
}
return schema
}

3
sql3/planner/planner.go
View file

@ -1,3 +0,0 @@
// Package planner contains everything required to build a query plan from a SQL
// statement.
package planner

228
sql3/planner/planoptimizer.go
View file

@ -8,10 +8,9 @@ import (
"reflect"
"strings"
"github.com/featurebasedb/featurebase/v3/errors"
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
"github.com/molecula/featurebase/v3/sql3"
"github.com/molecula/featurebase/v3/sql3/parser"
"github.com/molecula/featurebase/v3/sql3/planner/types"
)
//TODO(pok) push order by down as far as possible
@ -41,15 +40,12 @@ var optimizerFunctions = []OptimizerFunc{
// take the join out and use the appropriate PQL operator instead
tryToRewriteSubtableJoins,
// update the columnIdx for all the qualified references in various operators
fixFieldRefs,
// update the columnIdx for all the references in the projections
// based on the child operator for a projection
fixGroupByProjections,
// update the columnIdx for all the references in joins
fixJoinFieldRefs,
// update the columnIdx for all the references in group bys
fixGroupByFieldRefs,
fixProjectionReferences,
// if the query has one TableScanOperator then push the top
// expression down into that operator
@ -64,12 +60,12 @@ type OptimizerScope struct {
// optimizePlan takes a plan from the compiler and executes a series of transforms on it to optimize it
func (p *ExecutionPlanner) optimizePlan(ctx context.Context, plan types.PlanOperator) (types.PlanOperator, error) {
//log.Println("================================================================================")
//log.Println("plan pre-optimzation")
//jplan := plan.Plan()
//a, _ := json.MarshalIndent(jplan, "", " ")
//log.Println(string(a))
//log.Println("--------------------------------------------------------------------------------")
// log.Println("================================================================================")
// log.Println("plan pre-optimzation")
// jplan := plan.Plan()
// a, _ := json.MarshalIndent(jplan, "", " ")
// log.Println(string(a))
// log.Println("--------------------------------------------------------------------------------")
var err error
var result = plan
@ -79,6 +75,14 @@ func (p *ExecutionPlanner) optimizePlan(ctx context.Context, plan types.PlanOper
return nil, err
}
}
// log.Println("================================================================================")
// log.Println("plan ppst-optimzation")
// jplan = plan.Plan()
// a, _ = json.MarshalIndent(jplan, "", " ")
// log.Println(string(a))
// log.Println("--------------------------------------------------------------------------------")
return result, nil
}
@ -297,6 +301,7 @@ func pushdownFiltersToFilterableRelations(ctx context.Context, a *ExecutionPlann
}
filters.markFiltersHandled(tableFilters...)
// fix the field refs
tableFilters, _, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, tableNode.Schema(), tableFilters...)
if err != nil {
return nil, true, err
@ -320,6 +325,7 @@ func pushdownFiltersToAboveRelation(ctx context.Context, a *ExecutionPlanner, ta
if tableFilters := filters.availableFiltersForTable(table.Name()); len(tableFilters) > 0 {
filters.markFiltersHandled(tableFilters...)
// fix the field refs
handled, _, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, tableNode.Schema(), tableFilters...)
if err != nil {
return nil, true, err
@ -355,24 +361,16 @@ func pushdownFilters(ctx context.Context, a *ExecutionPlanner, n types.PlanOpera
if err != nil {
return nil, true, err
}
n, sameFix, err := fixFieldRefIndexesForOperator(ctx, a, n, scope)
if err != nil {
return nil, true, err
}
return n, samePred && sameFix, nil
return n, samePred, nil
case *PlanOpRelAlias, *PlanOpPQLTableScan:
n, samePred, err := pushdownFiltersToFilterableRelations(ctx, a, node, scope, filters, tableAliases)
if err != nil {
return nil, true, err
}
n, sameFix, err := fixFieldRefIndexesForOperator(ctx, a, n, scope)
if err != nil {
return nil, true, err
}
return n, samePred && sameFix, nil
return n, samePred, nil
default:
return fixFieldRefIndexesForOperator(ctx, a, node, scope)
return node, true, nil
}
})
}
@ -388,26 +386,18 @@ func pushdownFilters(ctx context.Context, a *ExecutionPlanner, n types.PlanOpera
if same {
return n, true, nil
}
n, _, err = fixFieldRefIndexesForOperator(ctx, a, n, scope)
if err != nil {
return nil, true, err
}
return n, false, nil
case *PlanOpRelAlias, *PlanOpPQLTableScan:
table, same, err := pushdownFiltersToAboveRelation(ctx, a, node, scope, filters)
_, same, err := pushdownFiltersToAboveRelation(ctx, a, node, scope, filters)
if err != nil {
return nil, true, err
}
if same {
return node, true, nil
}
node, _, err = fixFieldRefIndexesForOperator(ctx, a, table, scope)
if err != nil {
return nil, true, err
}
return node, false, nil
default:
return fixFieldRefIndexesForOperator(ctx, a, node, scope)
return node, true, nil
}
})
}
@ -730,23 +720,30 @@ func areAggregablesEqual(lhs types.Aggregable, rhs types.Aggregable) bool {
return false
}
func fixGroupByProjections(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
// fixes references for a projection op depending on child
func fixProjectionReferences(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) {
switch n := node.(type) {
case *PlanOpProjection:
switch childOp := n.ChildOp.(type) {
case *PlanOpGroupBy:
//PlanOpGroupBy's iterator returns group by exprs, then aggregates in the order they appear
childSchema := childOp.Schema()
for idx, pj := range n.Projections {
expr, _, err := TransformExpr(pj, func(e types.PlanExpression) (types.PlanExpression, bool, error) {
switch e.(type) {
switch thisAggregate := e.(type) {
case types.Aggregable:
// if we have a Aggregable, the AggExpression() will be a qualified ref
// given we are in the context of a PlanOpProjection with a PlanOpGroupBy
// we can use the ordinal position of the projection as the column index
ae := newQualifiedRefPlanExpression("", "", idx, e.Type())
return ae, false, nil
for idx, sc := range childSchema {
if strings.EqualFold(thisAggregate.String(), sc.ColumnName) {
ae := newQualifiedRefPlanExpression("", "", idx, e.Type())
return ae, false, nil
}
}
return nil, true, sql3.NewErrColumnNotFound(0, 0, thisAggregate.String())
default:
return e, true, nil
}
@ -893,40 +890,75 @@ func fixGroupByProjections(ctx context.Context, a *ExecutionPlanner, n types.Pla
n.Projections[idx] = expr
}
return n, false, nil
// everything else that can be a child of projection
case *PlanOpRelAlias, *PlanOpFilter, *PlanOpPQLTableScan, *PlanOpNestedLoops:
exprs, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, childOp.Schema(), n.Projections...)
if err != nil {
return n, true, err
}
n.Projections = exprs
return n, same, err
default:
return n, true, nil
}
return n, true, nil
default:
return n, true, nil
}
})
}
func fixJoinFieldRefs(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
func fixFieldRefs(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) {
switch n := node.(type) {
switch thisNode := node.(type) {
case *PlanOpFilter:
schema := thisNode.Schema()
expressions := thisNode.Expressions()
fixed, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, expressions...)
if err != nil {
return nil, true, err
}
newNode, err := thisNode.WithUpdatedExpressions(fixed...)
if err != nil {
return nil, true, err
}
return newNode, same, nil
case *PlanOpNestedLoops:
_, _, err := fixFieldRefIndexesForOperator(ctx, a, n, scope)
schema := thisNode.Schema()
expressions := thisNode.Expressions()
fixed, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, expressions...)
if err != nil {
return nil, true, err
}
return n, true, nil
default:
return n, true, nil
}
})
}
newNode, err := thisNode.WithUpdatedExpressions(fixed...)
if err != nil {
return nil, true, err
}
return newNode, same, nil
func fixGroupByFieldRefs(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) {
switch n := node.(type) {
case *PlanOpGroupBy:
_, _, err := fixFieldRefIndexesForOperator(ctx, a, n, scope)
schema := thisNode.ChildOp.Schema()
aggregateExpressions := thisNode.Aggregates
fixedAggregateExpressions, aggregateSame, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, aggregateExpressions...)
if err != nil {
return nil, true, err
}
return n, true, nil
groupByExpressions := thisNode.GroupByExprs
fixedGroupByExpressions, groupBySame, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, groupByExpressions...)
if err != nil {
return nil, true, err
}
newNode := NewPlanOpGroupBy(fixedAggregateExpressions, fixedGroupByExpressions, thisNode.ChildOp)
newNode.warnings = append(newNode.warnings, thisNode.warnings...)
return newNode, aggregateSame && groupBySame, nil
default:
return n, true, nil
return node, true, nil
}
})
}
@ -1015,24 +1047,35 @@ func getNestedLoopOperators(ctx context.Context, a *ExecutionPlanner, n types.Pl
func fixFieldRefIndexes(ctx context.Context, scope *OptimizerScope, a *ExecutionPlanner, schema types.Schema, exp types.PlanExpression) (types.PlanExpression, bool, error) {
return TransformExpr(exp, func(e types.PlanExpression) (types.PlanExpression, bool, error) {
switch e := e.(type) {
switch typedExpr := e.(type) {
case *qualifiedRefPlanExpression:
for i, col := range schema {
newIndex := i
if e.Name() == col.ColumnName && e.tableName == col.RelationName {
if newIndex != e.columnIndex {
// update the column index
return newQualifiedRefPlanExpression(e.tableName, e.columnName, newIndex, e.dataType), false, nil
if strings.EqualFold(typedExpr.Name(), col.ColumnName) {
if len(typedExpr.tableName) > 0 { // do we have a qualifier?
if typedExpr.tableName == col.RelationName || typedExpr.tableName == col.AliasName {
if newIndex != typedExpr.columnIndex {
// update the column index
return newQualifiedRefPlanExpression(typedExpr.tableName, typedExpr.columnName, newIndex, typedExpr.dataType), false, nil
}
return e, true, nil
}
} else { // no qualifier
if newIndex != typedExpr.columnIndex {
// update the column index
return newQualifiedRefPlanExpression(typedExpr.tableName, typedExpr.columnName, newIndex, typedExpr.dataType), false, nil
}
return e, true, nil
}
return e, true, nil
}
}
return nil, true, sql3.NewErrColumnNotFound(0, 0, e.Name())
return nil, true, sql3.NewErrColumnNotFound(0, 0, typedExpr.Name())
}
return e, true, nil
})
}
// for a list of expressions and an operator schema, fix the references for any qualifiedRef expressions
func fixFieldRefIndexesOnExpressions(ctx context.Context, scope *OptimizerScope, a *ExecutionPlanner, schema types.Schema, expressions ...types.PlanExpression) ([]types.PlanExpression, bool, error) {
var result []types.PlanExpression
var res types.PlanExpression
@ -1057,56 +1100,3 @@ func fixFieldRefIndexesOnExpressions(ctx context.Context, scope *OptimizerScope,
}
return expressions, true, nil
}
func fixFieldRefIndexesForOperator(ctx context.Context, a *ExecutionPlanner, node types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
if _, ok := node.(types.ContainsExpressions); !ok {
return node, true, nil
}
var schemas []types.Schema
for _, child := range node.Children() {
schemas = append(schemas, child.Schema())
}
if len(schemas) < 1 {
return node, true, nil
}
n, sameC, err := TransformPlanOpExprsWithPlanOp(node, func(_ types.PlanOperator, e types.PlanExpression) (types.PlanExpression, bool, error) {
for _, schema := range schemas {
fixed, same, err := fixFieldRefIndexes(ctx, scope, a, schema, e)
if err == nil {
return fixed, same, nil
}
if errors.Is(err, sql3.ErrColumnNotFound) {
continue
}
return nil, true, err
}
return e, true, nil
})
if err != nil {
return nil, true, err
}
sameJ := true
var cond types.PlanExpression
switch j := n.(type) {
case *PlanOpNestedLoops:
cond, sameJ, err = fixFieldRefIndexes(ctx, scope, a, j.Schema(), j.cond)
if err != nil {
return nil, true, err
}
if !sameJ {
n, err = j.WithUpdatedExpressions(cond)
if err != nil {
return nil, true, err
}
}
}
return n, sameC && sameJ, nil
}

47
sql3/planner/planwalker.go
View file

@ -156,12 +156,18 @@ func InspectExpressionsWithPlanOp(op types.PlanOperator, f exprWithNodeInspector
// If there was a transformation, the bool will be true, and an error if there was an error
type PlanOpTransformFunc func(op types.PlanOperator) (types.PlanOperator, bool, error)
// TransformPlanOp applies a transformation function to the given plan op graph
// TransformPlanOp applies a depth first transformation function to the given plan op
// It returns a tuple that is the result of the transformation; the new PlanOperator, a bool that
// is true if the resultant PlanOperator has not been transformed or an error.
// If the TransformPlanOp has children it will iterate the children and call the transformation
// function on each of them in turn. If those operators are transformed, it will create a new operator
// with those children. The last step is to call the transformation on the passed PlanOperator
func TransformPlanOp(op types.PlanOperator, f PlanOpTransformFunc) (types.PlanOperator, bool, error) {
thisOperator := op
children := op.Children()
children := thisOperator.Children()
if len(children) == 0 {
return f(op)
return f(thisOperator)
}
var newChildren []types.PlanOperator
@ -185,17 +191,17 @@ func TransformPlanOp(op types.PlanOperator, f PlanOpTransformFunc) (types.PlanOp
sameChildren := true
if len(newChildren) > 0 {
sameChildren = false
op, err = op.WithChildren(newChildren...)
thisOperator, err = thisOperator.WithChildren(newChildren...)
if err != nil {
return nil, true, err
}
}
op, sameOperator, err := f(op)
resultOperator, sameOperator, err := f(thisOperator)
if err != nil {
return nil, true, err
}
return op, sameChildren && sameOperator, nil
return resultOperator, sameChildren && sameOperator, nil
}
// ParentContext is a struct that enables transformation functions to include a parent operator
@ -209,6 +215,7 @@ type ParentContextFunc func(c ParentContext) (types.PlanOperator, bool, error)
type ParentSelectorFunc func(c ParentContext) bool
// TransformPlanOpWithParent applies a transformation function to a plan operator in the context that plan operators parent
func TransformPlanOpWithParent(op types.PlanOperator, s ParentSelectorFunc, f ParentContextFunc) (types.PlanOperator, bool, error) {
return planOpWithParentHelper(ParentContext{op, nil, -1}, s, f)
}
@ -252,11 +259,11 @@ func planOpWithParentHelper(c ParentContext, s ParentSelectorFunc, f ParentConte
}
}
operator, sameOperator, err := f(ParentContext{operator, c.Parent, c.ChildCount})
resultOperator, sameOperator, err := f(ParentContext{operator, c.Parent, c.ChildCount})
if err != nil {
return nil, true, err
}
return operator, sameChildren && sameOperator, nil
return resultOperator, sameChildren && sameOperator, nil
}
// ExprWithPlanOpFunc is a function that given an expression and the node
@ -362,11 +369,13 @@ func TransformSinglePlanOpExpressions(op types.PlanOperator, f ExprFunc) (types.
return op, true, nil
}
// TransformExpr applies a transformation function to an expression
// TransformExpr applies a depth first transformation function to an expression
func TransformExpr(expr types.PlanExpression, f ExprFunc) (types.PlanExpression, bool, error) {
thisExpr := expr
children := expr.Children()
if len(children) == 0 {
return f(expr)
return f(thisExpr)
}
var (
@ -392,22 +401,24 @@ func TransformExpr(expr types.PlanExpression, f ExprFunc) (types.PlanExpression,
sameChildren := true
if len(newChildren) > 0 {
sameChildren = false
expr, err = expr.WithChildren(newChildren...)
thisExpr, err = thisExpr.WithChildren(newChildren...)
if err != nil {
return nil, true, err
}
}
expr, sameExpr, err := f(expr)
resultExpr, sameExpr, err := f(thisExpr)
if err != nil {
return nil, true, err
}
return expr, sameChildren && sameExpr, nil
return resultExpr, sameChildren && sameExpr, nil
}
// TransformExprWithPlanOp applies a transformation function to an expression in the context of a plan operator
// TransformExprWithPlanOp applies a depth first transformation function to an expression in the context of a plan operator
func TransformExprWithPlanOp(n types.PlanOperator, e types.PlanExpression, f ExprWithPlanOpFunc) (types.PlanExpression, bool, error) {
children := e.Children()
thisExpr := e
children := thisExpr.Children()
if len(children) == 0 {
return f(n, e)
}
@ -435,15 +446,15 @@ func TransformExprWithPlanOp(n types.PlanOperator, e types.PlanExpression, f Exp
sameChilren := true
if len(newChildren) > 0 {
sameChilren = false
e, err = e.WithChildren(newChildren...)
thisExpr, err = thisExpr.WithChildren(newChildren...)
if err != nil {
return nil, true, err
}
}
e, sameExpr, err := f(n, e)
resultExpr, sameExpr, err := f(n, thisExpr)
if err != nil {
return nil, true, err
}
return e, sameChilren && sameExpr, nil
return resultExpr, sameChilren && sameExpr, nil
}

5
sql3/planner/types/planexpression.go
View file

@ -2,6 +2,7 @@ package types
import (
"context"
"fmt"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
)
@ -24,6 +25,8 @@ const (
// PlanExpression is an expression node for an execution plan
type PlanExpression interface {
fmt.Stringer
// evaluates expression based on current row
Evaluate(currentRow []interface{}) (interface{}, error)
@ -50,6 +53,8 @@ type AggregationBuffer interface {
// Interface to an expression that is a an aggregate
type Aggregable interface {
fmt.Stringer
NewBuffer() (AggregationBuffer, error)
AggType() AggregateFunctionType
AggExpression() PlanExpression

17
sql3/sql_complex_test.go
View file

@ -19,6 +19,23 @@ import (
"github.com/stretchr/testify/assert"
)
func TestPlanner_Misc(t *testing.T) {
d, err := parser.StringToDecimal("12.345678")
if err != nil {
t.Fatal(err)
}
assert.True(t, d.EqualTo(pql.NewDecimal(12345678, 6)))
d = parser.FloatToDecimalWithScale(12.345678, 6)
if err != nil {
t.Fatal(err)
}
assert.True(t, d.EqualTo(pql.NewDecimal(12345678, 6)))
}
func TestPlanner_Show(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()

49
sql3/test/defs/defs_join.go
View file

@ -1,5 +1,7 @@
package defs
import "github.com/molecula/featurebase/v3/pql"
// join tests
var joinTestsUsers = TableTest{
name: "jointestusers",
@ -54,10 +56,49 @@ var joinTests = TableTest{
hdr("", fldTypeDecimal2),
),
ExpRows: rows(
row(int64(1), float64(22.98)),
row(int64(0), float64(3.99)),
row(int64(2), float64(16.98)),
row(int64(3), float64(5.99)),
row(int64(1), pql.NewDecimal(2298, 2)),
row(int64(0), pql.NewDecimal(399, 2)),
row(int64(2), pql.NewDecimal(1698, 2)),
row(int64(3), pql.NewDecimal(599, 2)),
),
Compare: CompareExactOrdered,
},
{
name: "innerjoin-aggregate-groupby-sum-filter",
SQLs: sqls(
"select sum(price) from orders o inner join users u on o.userid = u._id where u.age > 20;",
),
ExpHdrs: hdrs(
hdr("", fldTypeDecimal2),
),
ExpRows: rows(
row(pql.NewDecimal(2696, 2)),
),
Compare: CompareExactOrdered,
},
{
name: "innerjoin-aggregate-groupby-count-distinct-filter",
SQLs: sqls(
"SELECT COUNT(DISTINCT u.name) FROM orders o JOIN users u ON o.userid = u._id WHERE o.price > 10;",
),
ExpHdrs: hdrs(
hdr("", fldTypeInt),
),
ExpRows: rows(
row(int64(2)),
),
Compare: CompareExactOrdered,
},
{
name: "innerjoin-aggregate-groupby-count-filter",
SQLs: sqls(
"SELECT COUNT(u.name) FROM orders o JOIN users u ON o.userid = u._id WHERE o.price > 10;",
),
ExpHdrs: hdrs(
hdr("", fldTypeInt),
),
ExpRows: rows(
row(int64(2)),
),
Compare: CompareExactOrdered,
},

10
sql3/test/defs/defs_unops.go
View file

@ -1,6 +1,10 @@
package defs
import "time"
import (
"time"
"github.com/molecula/featurebase/v3/pql"
)
var unaryOpExprWithInt = TableTest{
Table: tbl(
@ -157,7 +161,7 @@ var unaryOpExprWithDecimal = TableTest{
hdr("", fldTypeDecimal2),
),
ExpRows: rows(
row(float64(-12.34)),
row(pql.NewDecimal(-1234, 2)),
),
Compare: CompareExactUnordered,
},
@ -175,7 +179,7 @@ var unaryOpExprWithDecimal = TableTest{
hdr("", fldTypeDecimal2),
),
ExpRows: rows(
row(float64(12.34)),
row(pql.NewDecimal(1234, 2)),
),
Compare: CompareExactUnordered,
},

3
sql3/test/helpers.go
View file

@ -20,6 +20,9 @@ func MustQueryRows(tb testing.TB, svr *pilosa.Server, q string) ([][]interface{}
return nil, nil, err
}
// get the plan so that code runs during testing
_ = stmt.Plan()
ocolumns := stmt.Schema()
rowIter, err := stmt.Iterator(ctx, nil)