scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fixed join bugs by fixing query optimizer (fb-1699, fb-1700) (#2307)

Browse source 
this commit changes the way the plan is retrieved; implements Stringer on types.PlanExpression in preparation for HAVING support; removes last vestiges internal float64 arithmetic; implements a filter on PlanOpFilter; fixes various bugs in the PlanOptimizer when rewriting qualified references

* fixed selects with unqualified identifiers

* handle bad and non-existent query param inputs more appropriately

* added test coverage for PlanExpression Stringer

Co-authored-by: Matthew Jaffee <jaffee@pilosa.com>
This commit is contained in:
pokeeffe-molecula 2022-11-21 14:31:45 -06:00 • committed by GitHub
parent 10e8aa5c45
commit 5f662d2bce
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 661 additions and 199 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
http_handler.go
View file

@ -1387,6 +1387,17 @@ func (h *Handler) writeBadRequest(w http.ResponseWriter, r *http.Request, err er
// handlePostSQL handles /sql requests.
func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
includePlan := false
includePlanValue := r.URL.Query().Get("plan")
if len(includePlanValue) > 0 {
var err error
includePlan, err = strconv.ParseBool(includePlanValue)
if err != nil {
h.writeBadRequest(w, r, err)
return
}
}
b, err := io.ReadAll(r.Body)
@ -1455,6 +1466,17 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
}
}
writePlan := func(plan map[string]interface{}) {
if plan != nil && includePlan {
planBytes, err := json.Marshal(plan)
if err != nil {
planBytes = []byte(`"PROBLEM ENCODING QUERY PLAN"`)
}
w.Write([]byte(`,"queryPlan":`))
w.Write(planBytes)
}
}
// Get a query iterator.
iter, err := rootOperator.Iterator(r.Context(), nil)
if err != nil {
@ -1518,6 +1540,7 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
writeError(rowErr)
writeWarnings(rootOperator.Warnings())
writePlan(rootOperator.Plan())
}
type SQLField struct {

5
sql3/parser/astdatatype.go
View file

@ -257,6 +257,11 @@ func FloatToDecimal(v float64) pql.Decimal {
return pql.NewDecimal(unscaledValue, int64(scale))
}
func FloatToDecimalWithScale(v float64, s int64) pql.Decimal {
unscaledValue := int64(v * math.Pow(10, float64(s)))
return pql.NewDecimal(unscaledValue, int64(s))
}
func NumDecimalPlaces(v string) int {
i := strings.IndexByte(v, '.')
if i > -1 {

2
sql3/planner/executionplanner.go
View file

@ -79,12 +79,10 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen
default:
return nil, sql3.NewErrInternalf("cannot plan statement: %T", stmt)
}
// Optimize the plan.
if err == nil {
rootOperator, err = p.optimizePlan(ctx, rootOperator)
}
return rootOperator, err
}

208
sql3/planner/expression.go
View file

@ -166,6 +166,10 @@ func (n *unaryOpPlanExpression) Type() parser.ExprDataType {
return n.resultDataType
}
func (n *unaryOpPlanExpression) String() string {
return fmt.Sprintf("%s%s", n.op.String(), n.rhs.String())
}
func (n *unaryOpPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -233,7 +237,11 @@ func (n *unaryOpPlanExpression) plusWithTypeCheck(rhs interface{}) (interface{},
case *parser.DataTypeDecimal:
nr, nrok := rhs.(pql.Decimal)
if nrok {
return +(nr.Float64()), nil
val := nr.Value()
if !val.IsInt64() {
return nil, sql3.NewErrInternalf("decimal value overflow: %v", rhs)
}
return pql.NewDecimal(+val.Int64(), nr.Scale), nil
}
return nil, sql3.NewErrInternalf("unexpected incompatible types '%T", rhs)
@ -266,7 +274,12 @@ func (n *unaryOpPlanExpression) minusWithTypeCheck(rhs interface{}) (interface{}
case *parser.DataTypeDecimal:
nr, nrok := rhs.(pql.Decimal)
if nrok {
return -(nr.Float64()), nil
val := nr.Value()
if !val.IsInt64() {
return nil, sql3.NewErrInternalf("decimal value overflow: %v", rhs)
}
return pql.NewDecimal(-val.Int64(), nr.Scale), nil
}
return nil, sql3.NewErrInternalf("unexpected incompatible types '%T", rhs)
@ -644,6 +657,10 @@ func (n *binOpPlanExpression) Type() parser.ExprDataType {
return n.resultDataType
}
func (n *binOpPlanExpression) String() string {
return fmt.Sprintf("%s%s%s", n.lhs.String(), n.op.String(), n.rhs.String())
}
func (n *binOpPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -711,6 +728,10 @@ func (n *rangePlanExpression) Type() parser.ExprDataType {
return n.resultDataType
}
func (n *rangePlanExpression) String() string {
return fmt.Sprintf("between %s and %s", n.lhs.String(), n.rhs.String())
}
func (n *rangePlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -900,6 +921,24 @@ func (n *casePlanExpression) Type() parser.ExprDataType {
return n.resultDataType
}
func (n *casePlanExpression) String() string {
var result string
if n.baseExpr != nil {
result = fmt.Sprintf("case %s", n.baseExpr)
} else {
result = "case"
}
for _, blk := range n.blocks {
result += fmt.Sprintf(" %s", blk.String())
}
if n.elseExpr != nil {
result += fmt.Sprintf(" else %s end", n.elseExpr)
} else {
result += " end"
}
return result
}
func (n *casePlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -919,11 +958,45 @@ func (n *casePlanExpression) Plan() map[string]interface{} {
}
func (n *casePlanExpression) Children() []types.PlanExpression {
return []types.PlanExpression{}
result := make([]types.PlanExpression, 0)
if n.baseExpr != nil {
result = append(result, n.baseExpr)
}
result = append(result, n.blocks...)
if n.elseExpr != nil {
result = append(result, n.elseExpr)
}
return result
}
func (n *casePlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
return n, nil
currentLen := 0
if n.baseExpr != nil {
currentLen += 1
}
currentLen += len(n.blocks)
if n.elseExpr != nil {
currentLen += 1
}
if len(children) != currentLen {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
offset := 0
var newBaseExpr types.PlanExpression
if n.baseExpr != nil {
newBaseExpr = children[offset]
offset += 1
}
newBlocks := make([]types.PlanExpression, len(n.blocks))
copy(newBlocks[0:], children[offset:offset+len(n.blocks)])
offset += len(n.blocks)
var newElseExpr types.PlanExpression
if n.elseExpr != nil {
newElseExpr = children[offset]
}
return newCasePlanExpression(newBaseExpr, newBlocks, newElseExpr, n.resultDataType), nil
}
// caseBlockPlanExpression is for case blocks
@ -947,6 +1020,10 @@ func (n *caseBlockPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeBool()
}
func (n *caseBlockPlanExpression) String() string {
return fmt.Sprintf("when %s then %s end", n.condition.String(), n.body.String())
}
func (n *caseBlockPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -964,7 +1041,10 @@ func (n *caseBlockPlanExpression) Children() []types.PlanExpression {
}
func (n *caseBlockPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
return n, nil
if len(children) != 2 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newCaseBlockPlanExpression(children[0], children[1]), nil
}
// subqueryPlanExpression is a select statement (when used in an expression)
@ -1009,6 +1089,10 @@ func (n *subqueryPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeBool()
}
func (n *subqueryPlanExpression) String() string {
return n.op.String()
}
func (n *subqueryPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1110,6 +1194,13 @@ func (n *betweenOpPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeBool()
}
func (n *betweenOpPlanExpression) String() string {
if n.op == parser.BETWEEN {
return fmt.Sprintf("between %s and %s", n.lhs.String(), n.rhs.String())
}
return fmt.Sprintf("not between %s and %s", n.lhs.String(), n.rhs.String())
}
func (n *betweenOpPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1127,7 +1218,7 @@ func (n *betweenOpPlanExpression) Children() []types.PlanExpression {
}
func (n *betweenOpPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
if len(children) != 1 {
if len(children) != 2 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newBetweenOpPlanExpression(children[0], n.op, children[1]), nil
@ -1317,6 +1408,17 @@ func (n *inOpPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeBool()
}
func (n *inOpPlanExpression) String() string {
s := n.lhs.String()
if n.op == parser.NOTIN {
s += " not "
}
s += " in ("
s += n.rhs.String()
s += ")"
return s
}
func (n *inOpPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1335,7 +1437,7 @@ func (n *inOpPlanExpression) Children() []types.PlanExpression {
}
func (n *inOpPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
if len(children) != 1 {
if len(children) != 2 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newInOpPlanExpression(children[0], n.op, children[1]), nil
@ -1375,6 +1477,17 @@ func (n *callPlanExpression) Type() parser.ExprDataType {
return n.dataType
}
func (n *callPlanExpression) String() string {
args := ""
for idx, arg := range n.args {
if idx > 0 {
args += ", "
}
args += arg.String()
}
return fmt.Sprintf("%s(%s)", n.name, args)
}
func (n *callPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1427,6 +1540,10 @@ func (n *aliasPlanExpression) Type() parser.ExprDataType {
return n.expr.Type()
}
func (n *aliasPlanExpression) String() string {
return fmt.Sprintf("%s as %s", n.expr.String(), n.aliasName)
}
func (n *aliasPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1513,6 +1630,13 @@ func (n *qualifiedRefPlanExpression) Type() parser.ExprDataType {
return n.dataType
}
func (n *qualifiedRefPlanExpression) String() string {
if len(n.tableName) > 0 {
return fmt.Sprintf("%s.%s", n.tableName, n.columnName)
}
return n.columnName
}
func (n *qualifiedRefPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1571,6 +1695,10 @@ func (n *variableRefPlanExpression) Type() parser.ExprDataType {
return n.dataType
}
func (n *variableRefPlanExpression) String() string {
return fmt.Sprintf("@%s", n.name)
}
func (n *variableRefPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1602,6 +1730,10 @@ func (n *nullLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeVoid()
}
func (n *nullLiteralPlanExpression) String() string {
return "null"
}
func (n *nullLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1636,6 +1768,10 @@ func (n *intLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeInt()
}
func (n *intLiteralPlanExpression) String() string {
return fmt.Sprintf("%d", n.value)
}
func (n *intLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1678,6 +1814,10 @@ func (n *floatLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeDecimal(int64(scale))
}
func (n *floatLiteralPlanExpression) String() string {
return n.value
}
func (n *floatLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1713,6 +1853,10 @@ func (n *boolLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeBool()
}
func (n *boolLiteralPlanExpression) String() string {
return fmt.Sprintf("%v", n.value)
}
func (n *boolLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1748,6 +1892,10 @@ func (n *dateLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeTimestamp()
}
func (n *dateLiteralPlanExpression) String() string {
return n.value.Format(time.RFC3339Nano)
}
func (n *dateLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -1783,6 +1931,10 @@ func (n *stringLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeString()
}
func (n *stringLiteralPlanExpression) String() string {
return fmt.Sprintf("'%s'", n.value)
}
func (n *stringLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -2006,6 +2158,10 @@ func (n *castPlanExpression) Type() parser.ExprDataType {
return n.targetType
}
func (n *castPlanExpression) String() string {
return fmt.Sprintf("cast(%s as %s)", n.lhs.String(), n.targetType.TypeName())
}
func (n *castPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -2046,6 +2202,17 @@ func (n *exprListPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeVoid()
}
func (n *exprListPlanExpression) String() string {
var s string
for idx, expr := range n.exprs {
if idx > 0 {
s += ", "
}
s += expr.String()
}
return fmt.Sprintf("(%s)", s)
}
func (n *exprListPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -2062,7 +2229,10 @@ func (n *exprListPlanExpression) Children() []types.PlanExpression {
}
func (n *exprListPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
return n, nil
if len(children) != len(n.exprs) {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newExprListExpression(children), nil
}
// exprSetLiteralPlanExpression is a set literal
@ -2122,6 +2292,17 @@ func (n *exprSetLiteralPlanExpression) Type() parser.ExprDataType {
return n.dataType
}
func (n *exprSetLiteralPlanExpression) String() string {
var members string
for idx, m := range n.members {
if idx > 0 {
members += ", "
}
members += m.String()
}
return fmt.Sprintf("[%s]", members)
}
func (n *exprSetLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -2192,6 +2373,17 @@ func (n *exprTupleLiteralPlanExpression) Type() parser.ExprDataType {
return n.dataType
}
func (n *exprTupleLiteralPlanExpression) String() string {
members := ""
for idx, m := range n.members {
if idx > 0 {
members += ", "
}
members += m.String()
}
return fmt.Sprintf("{%s}", members)
}
func (n *exprTupleLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)

84
sql3/planner/expression_it_test.go Normal file
View file

@ -0,0 +1,84 @@
package planner
import (
"testing"
"time"
"github.com/molecula/featurebase/v3/sql3/parser"
"github.com/molecula/featurebase/v3/sql3/planner/types"
"github.com/stretchr/testify/assert"
)
func TestExpressions(t *testing.T) {
t.Run("StringerTest", func(t *testing.T) {
uop := newUnaryOpPlanExpression(parser.PLUS, newIntLiteralPlanExpression(10), parser.NewDataTypeInt())
assert.Equal(t, uop.String(), "+10")
bop := newBinOpPlanExpression(newIntLiteralPlanExpression(10), parser.PLUS, newIntLiteralPlanExpression(20), parser.NewDataTypeInt())
assert.Equal(t, bop.String(), "10+20")
rop := newRangeOpPlanExpression(newIntLiteralPlanExpression(10), newIntLiteralPlanExpression(20), parser.NewDataTypeInt())
assert.Equal(t, rop.String(), "between 10 and 20")
cop := newCasePlanExpression(newStringLiteralPlanExpression("foo"),
[]types.PlanExpression{
newCaseBlockPlanExpression(newStringLiteralPlanExpression("20"), newIntLiteralPlanExpression(20)),
newCaseBlockPlanExpression(newStringLiteralPlanExpression("30"), newIntLiteralPlanExpression(20)),
},
newIntLiteralPlanExpression(20), parser.NewDataTypeInt())
assert.Equal(t, cop.String(), "case 'foo' when '20' then 20 end when '30' then 20 end else 20 end")
bwop := newBetweenOpPlanExpression(newIntLiteralPlanExpression(10), parser.BETWEEN, newIntLiteralPlanExpression(20))
assert.Equal(t, bwop.String(), "between 10 and 20")
iop := newInOpPlanExpression(newIntLiteralPlanExpression(10), parser.IN, newIntLiteralPlanExpression(20))
assert.Equal(t, iop.String(), "10 in (20)")
callop := newCallPlanExpression("foo", []types.PlanExpression{newIntLiteralPlanExpression(10)}, parser.NewDataTypeInt())
assert.Equal(t, callop.String(), "foo(10)")
alop := newAliasPlanExpression("frobny", newIntLiteralPlanExpression(10))
assert.Equal(t, alop.String(), "10 as frobny")
qrop := newQualifiedRefPlanExpression("foo", "bar", 1, parser.NewDataTypeInt())
assert.Equal(t, qrop.String(), "foo.bar")
vop := newVariableRefPlanExpression("foo", 1, parser.NewDataTypeInt())
assert.Equal(t, vop.String(), "@foo")
nulop := newNullLiteralPlanExpression()
assert.Equal(t, nulop.String(), "null")
ilop := newIntLiteralPlanExpression(10)
assert.Equal(t, ilop.String(), "10")
flop := newFloatLiteralPlanExpression("12.3456")
assert.Equal(t, flop.String(), "12.3456")
blop := newBoolLiteralPlanExpression(false)
assert.Equal(t, blop.String(), "false")
tm, _ := time.ParseInLocation(time.RFC3339, "2012-11-01T22:08:41+00:00", time.UTC)
dlop := newDateLiteralPlanExpression(tm)
assert.Equal(t, dlop.String(), "2012-11-01T22:08:41Z")
slop := newStringLiteralPlanExpression("foo")
assert.Equal(t, slop.String(), "'foo'")
ctop := newCastPlanExpression(newIntLiteralPlanExpression(10), parser.NewDataTypeString())
assert.Equal(t, ctop.String(), "cast(10 as STRING)")
elop := newExprListExpression([]types.PlanExpression{newStringLiteralPlanExpression("foo"), newStringLiteralPlanExpression("bar")})
assert.Equal(t, elop.String(), "('foo', 'bar')")
stlop := newExprSetLiteralPlanExpression([]types.PlanExpression{newStringLiteralPlanExpression("foo"), newStringLiteralPlanExpression("bar")}, parser.NewDataTypeString())
assert.Equal(t, stlop.String(), "['foo', 'bar']")
tplop := newExprTupleLiteralPlanExpression([]types.PlanExpression{newStringLiteralPlanExpression("foo"), newStringLiteralPlanExpression("bar")}, parser.NewDataTypeString())
assert.Equal(t, tplop.String(), "{'foo', 'bar'}")
})
}

64
sql3/planner/expressionagg.go
View file

@ -120,6 +120,10 @@ func (n *countPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *countPlanExpression) String() string {
return fmt.Sprintf("count(%s)", n.arg.String())
}
func (n *countPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -179,6 +183,10 @@ func (n *countDistinctPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *countDistinctPlanExpression) String() string {
return fmt.Sprintf("count(distinct %s)", n.arg.String())
}
func (n *countDistinctPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -197,13 +205,12 @@ func (n *countDistinctPlanExpression) WithChildren(children ...types.PlanExpress
// aggregator for the SUM function
type aggregateSum struct {
sum float64
sum interface{}
expr types.PlanExpression
}
func NewAggSumBuffer(child types.PlanExpression) *aggregateSum {
return &aggregateSum{
sum: float64(0),
expr: child,
}
}
@ -230,7 +237,17 @@ func (m *aggregateSum) Update(ctx context.Context, row types.Row) error {
if !ok {
return sql3.NewErrInternalf("unexpected type conversion '%T'", v)
}
m.sum += val.Float64()
var dsum pql.Decimal
if m.sum != nil {
dsum, ok = m.sum.(pql.Decimal)
if !ok {
return sql3.NewErrInternalf("unexpected type conversion '%T'", m.sum)
}
} else {
dsum = pql.NewDecimal(0, dataType.Scale)
}
dsum = pql.AddDecimal(dsum, val)
m.sum = dsum
default:
return sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", dataType)
}
@ -238,7 +255,17 @@ func (m *aggregateSum) Update(ctx context.Context, row types.Row) error {
}
func (m *aggregateSum) Eval(ctx context.Context) (interface{}, error) {
return m.sum, nil
switch m.expr.Type().(type) {
case *parser.DataTypeDecimal:
dsum, ok := m.sum.(pql.Decimal)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type conversion '%T'", m.sum)
}
return dsum, nil
default:
return nil, sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", m.expr.Type())
}
}
// sumPlanExpression handles SUM()
@ -284,6 +311,10 @@ func (n *sumPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *sumPlanExpression) String() string {
return fmt.Sprintf("sum(%s)", n.arg.String())
}
func (n *sumPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -293,11 +324,16 @@ func (n *sumPlanExpression) Plan() map[string]interface{} {
}
func (n *sumPlanExpression) Children() []types.PlanExpression {
return nil
return []types.PlanExpression{
n.arg,
}
}
func (n *sumPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
return n, nil
if len(children) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newSumPlanExpression(children[0], n.returnDataType), nil
}
// aggregator for AVG
@ -387,6 +423,10 @@ func (n *avgPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *avgPlanExpression) String() string {
return fmt.Sprintf("avg(%s)", n.arg.String())
}
func (n *avgPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -480,6 +520,10 @@ func (n *minPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *minPlanExpression) String() string {
return fmt.Sprintf("min(%s)", n.arg.String())
}
func (n *minPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -573,6 +617,10 @@ func (n *maxPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *maxPlanExpression) String() string {
return fmt.Sprintf("max(%s)", n.arg.String())
}
func (n *maxPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
@ -636,6 +684,10 @@ func (n *percentilePlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *percentilePlanExpression) String() string {
return fmt.Sprintf("percentile(%s)", n.arg.String())
}
func (n *percentilePlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)

2
sql3/planner/expressionanalyzer.go
View file

@ -337,7 +337,7 @@ func (p *ExecutionPlanner) analyzeUnaryExpression(expr *parser.UnaryExpr, scope
}
if typeIsInteger(x.DataType()) {
expr.ResultDataType = parser.NewDataTypeInt()
} else if typeIsFloat(x.DataType()) {
} else if typeIsDecimal(x.DataType()) {
fd, ok := x.DataType().(*parser.DataTypeDecimal)
if !ok {
return nil, sql3.NewErrInternalf("unexpected data type")

8
sql3/planner/expressionanalyzercall.go
View file

@ -67,7 +67,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser
}
//make sure the ref is sum-able
if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType())) {
if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType())) {
return nil, sql3.NewErrIntOrDecimalExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column)
}
@ -95,7 +95,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser
}
//make sure the ref is avg-able
if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType())) {
if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType())) {
return nil, sql3.NewErrIntOrDecimalExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column)
}
@ -123,7 +123,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser
}
//make sure the ref is percentilable-able
if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType()) || typeIsTimestamp(ref.DataType())) {
if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType()) || typeIsTimestamp(ref.DataType())) {
return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column)
}
@ -164,7 +164,7 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser
}
// make sure the ref is min/max-able
if !(typeIsInteger(ref.DataType()) || typeIsFloat(ref.DataType()) || typeIsTimestamp(ref.DataType())) {
if !(typeIsInteger(ref.DataType()) || typeIsDecimal(ref.DataType()) || typeIsTimestamp(ref.DataType())) {
return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(ref.Table.NamePos.Line, ref.Table.NamePos.Column)
}

4
sql3/planner/expressiontypes.go
View file

@ -481,8 +481,8 @@ func typeIsTimestamp(testType parser.ExprDataType) bool {
}
}
// returns true if the type is a float
func typeIsFloat(testType parser.ExprDataType) bool {
// returns true if the type is a decimal
func typeIsDecimal(testType parser.ExprDataType) bool {
switch testType.(type) {
case *parser.DataTypeDecimal:
return true

34
sql3/planner/opfilter.go
View file

@ -61,6 +61,7 @@ func (p *PlanOpFilter) Plan() map[string]interface{} {
ps = append(ps, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeName()))
}
result["_schema"] = ps
result["predicate"] = p.Predicate.Plan()
result["child"] = p.ChildOp.Plan()
return result
}
@ -77,6 +78,23 @@ func (p *PlanOpFilter) Warnings() []string {
return p.warnings
}
func (p *PlanOpFilter) Expressions() []types.PlanExpression {
if p.Predicate != nil {
return []types.PlanExpression{
p.Predicate,
}
}
return []types.PlanExpression{}
}
func (p *PlanOpFilter) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) {
if len(exprs) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs))
}
p.Predicate = exprs[0]
return p, nil
}
type filterIterator struct {
predicate types.PlanExpression
child types.RowIterator
@ -92,6 +110,18 @@ func newFilterIterator(ctx context.Context, predicate types.PlanExpression, chil
}
func (i *filterIterator) Next(ctx context.Context) (types.Row, error) {
//TODO (pok) - actually implement the filter
return i.child.Next(ctx)
for {
row, err := i.child.Next(ctx)
if err != nil {
return nil, err
}
matches, err := conditionIsTrue(ctx, row, i.predicate)
if err != nil {
return nil, err
}
if !matches {
continue
}
return row, nil
}
}

7
sql3/planner/opgroupby.go
View file

@ -49,7 +49,7 @@ func (p *PlanOpGroupBy) Schema() types.Schema {
offset := len(p.GroupByExprs)
for idx, agg := range p.Aggregates {
s := &types.PlannerColumn{
ColumnName: "",
ColumnName: agg.String(),
RelationName: "",
Type: agg.Type(),
}
@ -86,7 +86,7 @@ func (p *PlanOpGroupBy) WithChildren(children ...types.PlanOperator) (types.Plan
func (p *PlanOpGroupBy) Expressions() []types.PlanExpression {
result := []types.PlanExpression{}
result = append(result, p.GroupByExprs...)
result = append(result, p.Aggregates...)
return result
}
@ -94,8 +94,7 @@ func (p *PlanOpGroupBy) WithUpdatedExpressions(exprs ...types.PlanExpression) (t
if len(exprs) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs))
}
p.GroupByExprs = exprs
return p, nil
return NewPlanOpGroupBy(exprs, p.GroupByExprs, p.ChildOp), nil
}
func (p *PlanOpGroupBy) Plan() map[string]interface{} {

17
sql3/planner/opnestedloops.go
View file

@ -38,6 +38,7 @@ func (p *PlanOpNestedLoops) Plan() map[string]interface{} {
result["_schema"] = ps
result["top"] = p.top.Plan()
result["bottom"] = p.bottom.Plan()
result["condition"] = p.cond.Plan()
return result
}
@ -124,8 +125,6 @@ type nestedLoopsIter struct {
rowSize int
originalRow types.Row
bottomRows RowCache
}
func newNestedLoopsIter(ctx context.Context, jt joinType, top types.RowIterator, bottom types.RowIterable, scopeRow types.Row, joinCondition types.PlanExpression, rowWidth int, originalRow types.Row) *nestedLoopsIter {
@ -136,7 +135,6 @@ func newNestedLoopsIter(ctx context.Context, jt joinType, top types.RowIterator,
cond: joinCondition,
rowSize: rowWidth,
originalRow: originalRow,
bottomRows: newInMemoryRowCache(),
ctx: ctx,
}
}
@ -153,11 +151,14 @@ func (i *nestedLoopsIter) loadTop(ctx context.Context) error {
i.topRow = i.originalRow.Append(r)
i.foundMatch = false
//DEBUG log.Printf("top row %v", i.topRow)
return nil
}
func (i *nestedLoopsIter) loadBottom(ctx context.Context) (row types.Row, err error) {
if i.bottom == nil {
// DEBUG log.Printf("bottom row initializing iterator...")
var iter types.RowIterator
iter, err = i.bottomProvider.Iterator(ctx, i.topRow)
if err != nil {
@ -169,16 +170,15 @@ func (i *nestedLoopsIter) loadBottom(ctx context.Context) (row types.Row, err er
rightRow, err := i.bottom.Next(ctx)
if err != nil {
if err == types.ErrNoMoreRows {
// DEBUG log.Printf("bottom end of rows")
i.bottom = nil
i.topRow = nil
return nil, types.ErrNoMoreRows
}
return nil, err
}
err = i.bottomRows.Add(rightRow)
if err != nil {
return nil, err
}
//DEBUG log.Printf("bottom row %v", rightRow)
return rightRow, nil
}
@ -247,7 +247,8 @@ func (i *nestedLoopsIter) Next(ctx context.Context) (types.Row, error) {
i.foundMatch = true
//DEBUG log.Printf("Join result %v", row)
// DEBUG log.Printf("join result %v", row)
return row, nil
}
}

36
sql3/planner/oppqltablescan.go
View file

@ -85,12 +85,17 @@ func (p *PlanOpPQLTableScan) Schema() types.Schema {
return result
}
for _, col := range table.Fields {
result = append(result, &types.PlannerColumn{
ColumnName: col.Name,
RelationName: p.tableName,
Type: fieldSQLDataType(col),
})
for _, col := range p.columns {
for _, fld := range table.Fields {
if strings.EqualFold(fld.Name, col) {
result = append(result, &types.PlannerColumn{
ColumnName: fld.Name,
RelationName: p.tableName,
Type: fieldSQLDataType(fld),
})
break
}
}
}
return result
}
@ -149,15 +154,20 @@ func (i *tableScanRowIter) Next(ctx context.Context) (types.Row, error) {
}
return nil, err
}
i.rowWidth = len(table.Fields)
i.rowWidth = len(i.columns)
i.columnMap = make(map[string]*targetColumn)
for idx, fld := range table.Fields {
i.columnMap[fld.Name] = &targetColumn{
columnIdx: idx,
srcColumnIdx: -1,
columnName: fld.Name,
dataType: fieldSQLDataType(fld),
for idx, col := range i.columns {
for _, fld := range table.Fields {
if strings.EqualFold(col, fld.Name) {
i.columnMap[fld.Name] = &targetColumn{
columnIdx: idx,
srcColumnIdx: -1,
columnName: fld.Name,
dataType: fieldSQLDataType(fld),
}
break
}
}
}

7
sql3/planner/opprojection.go
View file

@ -60,12 +60,12 @@ func (p *PlanOpProjection) WithChildren(children ...types.PlanOperator) (types.P
func (p *PlanOpProjection) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_op"] = fmt.Sprintf("%T", p)
result["__op"] = fmt.Sprintf("%T", p)
sc := make([]string, 0)
for _, e := range p.Schema() {
sc = append(sc, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeName()))
}
result["_schema"] = sc
result["__schema"] = sc
result["child"] = p.ChildOp.Plan()
@ -73,7 +73,7 @@ func (p *PlanOpProjection) Plan() map[string]interface{} {
for _, e := range p.Projections {
ps = append(ps, e.Plan())
}
result["projections"] = ps
result["_projections"] = ps
return result
}
@ -100,7 +100,6 @@ func ExpressionToColumn(e types.PlanExpression) *types.PlannerColumn {
if n, ok := e.(types.IdentifiableByName); ok {
name = n.Name()
} else {
//TODO(pok) - implement this
name = "" //e.String()
}

1
sql3/planner/opquery.go
View file

@ -28,6 +28,7 @@ func NewPlanOpQuery(child types.PlanOperator, sql string) *PlanOpQuery {
return &PlanOpQuery{
ChildOp: child,
warnings: make([]string, 0),
sql: sql,
}
}

2
sql3/planner/oprelalias.go
View file

@ -28,7 +28,7 @@ func NewPlanOpRelAlias(alias string, child types.PlanOperator) *PlanOpRelAlias {
func (p *PlanOpRelAlias) Schema() types.Schema {
schema := p.ChildOp.Schema()
for _, s := range schema {
s.RelationName = p.alias
s.AliasName = p.alias
}
return schema
}

3
sql3/planner/planner.go
View file

@ -1,3 +0,0 @@
// Package planner contains everything required to build a query plan from a SQL
// statement.
package planner

222
sql3/planner/planoptimizer.go
View file

@ -8,7 +8,6 @@ import (
"reflect"
"strings"
"github.com/molecula/featurebase/v3/errors"
"github.com/molecula/featurebase/v3/sql3"
"github.com/molecula/featurebase/v3/sql3/parser"
"github.com/molecula/featurebase/v3/sql3/planner/types"
@ -41,15 +40,12 @@ var optimizerFunctions = []OptimizerFunc{
// take the join out and use the appropriate PQL operator instead
tryToRewriteSubtableJoins,
// update the columnIdx for all the qualified references in various operators
fixFieldRefs,
// update the columnIdx for all the references in the projections
// based on the child operator for a projection
fixGroupByProjections,
// update the columnIdx for all the references in joins
fixJoinFieldRefs,
// update the columnIdx for all the references in group bys
fixGroupByFieldRefs,
fixProjectionReferences,
// if the query has one TableScanOperator then push the top
// expression down into that operator
@ -64,12 +60,12 @@ type OptimizerScope struct {
// optimizePlan takes a plan from the compiler and executes a series of transforms on it to optimize it
func (p *ExecutionPlanner) optimizePlan(ctx context.Context, plan types.PlanOperator) (types.PlanOperator, error) {
//log.Println("================================================================================")
//log.Println("plan pre-optimzation")
//jplan := plan.Plan()
//a, _ := json.MarshalIndent(jplan, "", " ")
//log.Println(string(a))
//log.Println("--------------------------------------------------------------------------------")
// log.Println("================================================================================")
// log.Println("plan pre-optimzation")
// jplan := plan.Plan()
// a, _ := json.MarshalIndent(jplan, "", " ")
// log.Println(string(a))
// log.Println("--------------------------------------------------------------------------------")
var err error
var result = plan
@ -79,6 +75,14 @@ func (p *ExecutionPlanner) optimizePlan(ctx context.Context, plan types.PlanOper
return nil, err
}
}
// log.Println("================================================================================")
// log.Println("plan ppst-optimzation")
// jplan = plan.Plan()
// a, _ = json.MarshalIndent(jplan, "", " ")
// log.Println(string(a))
// log.Println("--------------------------------------------------------------------------------")
return result, nil
}
@ -297,6 +301,7 @@ func pushdownFiltersToFilterableRelations(ctx context.Context, a *ExecutionPlann
}
filters.markFiltersHandled(tableFilters...)
// fix the field refs
tableFilters, _, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, tableNode.Schema(), tableFilters...)
if err != nil {
return nil, true, err
@ -320,6 +325,7 @@ func pushdownFiltersToAboveRelation(ctx context.Context, a *ExecutionPlanner, ta
if tableFilters := filters.availableFiltersForTable(table.Name()); len(tableFilters) > 0 {
filters.markFiltersHandled(tableFilters...)
// fix the field refs
handled, _, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, tableNode.Schema(), tableFilters...)
if err != nil {
return nil, true, err
@ -355,24 +361,16 @@ func pushdownFilters(ctx context.Context, a *ExecutionPlanner, n types.PlanOpera
if err != nil {
return nil, true, err
}
n, sameFix, err := fixFieldRefIndexesForOperator(ctx, a, n, scope)
if err != nil {
return nil, true, err
}
return n, samePred && sameFix, nil
return n, samePred, nil
case *PlanOpRelAlias, *PlanOpPQLTableScan:
n, samePred, err := pushdownFiltersToFilterableRelations(ctx, a, node, scope, filters, tableAliases)
if err != nil {
return nil, true, err
}
n, sameFix, err := fixFieldRefIndexesForOperator(ctx, a, n, scope)
if err != nil {
return nil, true, err
}
return n, samePred && sameFix, nil
return n, samePred, nil
default:
return fixFieldRefIndexesForOperator(ctx, a, node, scope)
return node, true, nil
}
})
}
@ -388,26 +386,18 @@ func pushdownFilters(ctx context.Context, a *ExecutionPlanner, n types.PlanOpera
if same {
return n, true, nil
}
n, _, err = fixFieldRefIndexesForOperator(ctx, a, n, scope)
if err != nil {
return nil, true, err
}
return n, false, nil
case *PlanOpRelAlias, *PlanOpPQLTableScan:
table, same, err := pushdownFiltersToAboveRelation(ctx, a, node, scope, filters)
_, same, err := pushdownFiltersToAboveRelation(ctx, a, node, scope, filters)
if err != nil {
return nil, true, err
}
if same {
return node, true, nil
}
node, _, err = fixFieldRefIndexesForOperator(ctx, a, table, scope)
if err != nil {
return nil, true, err
}
return node, false, nil
default:
return fixFieldRefIndexesForOperator(ctx, a, node, scope)
return node, true, nil
}
})
}
@ -730,23 +720,30 @@ func areAggregablesEqual(lhs types.Aggregable, rhs types.Aggregable) bool {
return false
}
func fixGroupByProjections(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
// fixes references for a projection op depending on child
func fixProjectionReferences(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) {
switch n := node.(type) {
case *PlanOpProjection:
switch childOp := n.ChildOp.(type) {
case *PlanOpGroupBy:
//PlanOpGroupBy's iterator returns group by exprs, then aggregates in the order they appear
childSchema := childOp.Schema()
for idx, pj := range n.Projections {
expr, _, err := TransformExpr(pj, func(e types.PlanExpression) (types.PlanExpression, bool, error) {
switch e.(type) {
switch thisAggregate := e.(type) {
case types.Aggregable:
// if we have a Aggregable, the AggExpression() will be a qualified ref
// given we are in the context of a PlanOpProjection with a PlanOpGroupBy
// we can use the ordinal position of the projection as the column index
ae := newQualifiedRefPlanExpression("", "", idx, e.Type())
return ae, false, nil
for idx, sc := range childSchema {
if strings.EqualFold(thisAggregate.String(), sc.ColumnName) {
ae := newQualifiedRefPlanExpression("", "", idx, e.Type())
return ae, false, nil
}
}
return nil, true, sql3.NewErrColumnNotFound(0, 0, thisAggregate.String())
default:
return e, true, nil
}
@ -893,40 +890,75 @@ func fixGroupByProjections(ctx context.Context, a *ExecutionPlanner, n types.Pla
n.Projections[idx] = expr
}
return n, false, nil
// everything else that can be a child of projection
case *PlanOpRelAlias, *PlanOpFilter, *PlanOpPQLTableScan, *PlanOpNestedLoops:
exprs, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, childOp.Schema(), n.Projections...)
if err != nil {
return n, true, err
}
n.Projections = exprs
return n, same, err
default:
return n, true, nil
}
return n, true, nil
default:
return n, true, nil
}
})
}
func fixJoinFieldRefs(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
func fixFieldRefs(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) {
switch n := node.(type) {
switch thisNode := node.(type) {
case *PlanOpFilter:
schema := thisNode.Schema()
expressions := thisNode.Expressions()
fixed, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, expressions...)
if err != nil {
return nil, true, err
}
newNode, err := thisNode.WithUpdatedExpressions(fixed...)
if err != nil {
return nil, true, err
}
return newNode, same, nil
case *PlanOpNestedLoops:
_, _, err := fixFieldRefIndexesForOperator(ctx, a, n, scope)
schema := thisNode.Schema()
expressions := thisNode.Expressions()
fixed, same, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, expressions...)
if err != nil {
return nil, true, err
}
return n, true, nil
default:
return n, true, nil
}
})
}
newNode, err := thisNode.WithUpdatedExpressions(fixed...)
if err != nil {
return nil, true, err
}
return newNode, same, nil
func fixGroupByFieldRefs(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) {
switch n := node.(type) {
case *PlanOpGroupBy:
_, _, err := fixFieldRefIndexesForOperator(ctx, a, n, scope)
schema := thisNode.ChildOp.Schema()
aggregateExpressions := thisNode.Aggregates
fixedAggregateExpressions, aggregateSame, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, aggregateExpressions...)
if err != nil {
return nil, true, err
}
return n, true, nil
groupByExpressions := thisNode.GroupByExprs
fixedGroupByExpressions, groupBySame, err := fixFieldRefIndexesOnExpressions(ctx, scope, a, schema, groupByExpressions...)
if err != nil {
return nil, true, err
}
newNode := NewPlanOpGroupBy(fixedAggregateExpressions, fixedGroupByExpressions, thisNode.ChildOp)
newNode.warnings = append(newNode.warnings, thisNode.warnings...)
return newNode, aggregateSame && groupBySame, nil
default:
return n, true, nil
return node, true, nil
}
})
}
@ -1015,24 +1047,35 @@ func getNestedLoopOperators(ctx context.Context, a *ExecutionPlanner, n types.Pl
func fixFieldRefIndexes(ctx context.Context, scope *OptimizerScope, a *ExecutionPlanner, schema types.Schema, exp types.PlanExpression) (types.PlanExpression, bool, error) {
return TransformExpr(exp, func(e types.PlanExpression) (types.PlanExpression, bool, error) {
switch e := e.(type) {
switch typedExpr := e.(type) {
case *qualifiedRefPlanExpression:
for i, col := range schema {
newIndex := i
if e.Name() == col.ColumnName && e.tableName == col.RelationName {
if newIndex != e.columnIndex {
// update the column index
return newQualifiedRefPlanExpression(e.tableName, e.columnName, newIndex, e.dataType), false, nil
if strings.EqualFold(typedExpr.Name(), col.ColumnName) {
if len(typedExpr.tableName) > 0 { // do we have a qualifier?
if typedExpr.tableName == col.RelationName || typedExpr.tableName == col.AliasName {
if newIndex != typedExpr.columnIndex {
// update the column index
return newQualifiedRefPlanExpression(typedExpr.tableName, typedExpr.columnName, newIndex, typedExpr.dataType), false, nil
}
return e, true, nil
}
} else { // no qualifier
if newIndex != typedExpr.columnIndex {
// update the column index
return newQualifiedRefPlanExpression(typedExpr.tableName, typedExpr.columnName, newIndex, typedExpr.dataType), false, nil
}
return e, true, nil
}
return e, true, nil
}
}
return nil, true, sql3.NewErrColumnNotFound(0, 0, e.Name())
return nil, true, sql3.NewErrColumnNotFound(0, 0, typedExpr.Name())
}
return e, true, nil
})
}
// for a list of expressions and an operator schema, fix the references for any qualifiedRef expressions
func fixFieldRefIndexesOnExpressions(ctx context.Context, scope *OptimizerScope, a *ExecutionPlanner, schema types.Schema, expressions ...types.PlanExpression) ([]types.PlanExpression, bool, error) {
var result []types.PlanExpression
var res types.PlanExpression
@ -1057,56 +1100,3 @@ func fixFieldRefIndexesOnExpressions(ctx context.Context, scope *OptimizerScope,
}
return expressions, true, nil
}
func fixFieldRefIndexesForOperator(ctx context.Context, a *ExecutionPlanner, node types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) {
if _, ok := node.(types.ContainsExpressions); !ok {
return node, true, nil
}
var schemas []types.Schema
for _, child := range node.Children() {
schemas = append(schemas, child.Schema())
}
if len(schemas) < 1 {
return node, true, nil
}
n, sameC, err := TransformPlanOpExprsWithPlanOp(node, func(_ types.PlanOperator, e types.PlanExpression) (types.PlanExpression, bool, error) {
for _, schema := range schemas {
fixed, same, err := fixFieldRefIndexes(ctx, scope, a, schema, e)
if err == nil {
return fixed, same, nil
}
if errors.Is(err, sql3.ErrColumnNotFound) {
continue
}
return nil, true, err
}
return e, true, nil
})
if err != nil {
return nil, true, err
}
sameJ := true
var cond types.PlanExpression
switch j := n.(type) {
case *PlanOpNestedLoops:
cond, sameJ, err = fixFieldRefIndexes(ctx, scope, a, j.Schema(), j.cond)
if err != nil {
return nil, true, err
}
if !sameJ {
n, err = j.WithUpdatedExpressions(cond)
if err != nil {
return nil, true, err
}
}
}
return n, sameC && sameJ, nil
}

47
sql3/planner/planwalker.go
View file

@ -156,12 +156,18 @@ func InspectExpressionsWithPlanOp(op types.PlanOperator, f exprWithNodeInspector
// If there was a transformation, the bool will be true, and an error if there was an error
type PlanOpTransformFunc func(op types.PlanOperator) (types.PlanOperator, bool, error)
// TransformPlanOp applies a transformation function to the given plan op graph
// TransformPlanOp applies a depth first transformation function to the given plan op
// It returns a tuple that is the result of the transformation; the new PlanOperator, a bool that
// is true if the resultant PlanOperator has not been transformed or an error.
// If the TransformPlanOp has children it will iterate the children and call the transformation
// function on each of them in turn. If those operators are transformed, it will create a new operator
// with those children. The last step is to call the transformation on the passed PlanOperator
func TransformPlanOp(op types.PlanOperator, f PlanOpTransformFunc) (types.PlanOperator, bool, error) {
thisOperator := op
children := op.Children()
children := thisOperator.Children()
if len(children) == 0 {
return f(op)
return f(thisOperator)
}
var newChildren []types.PlanOperator
@ -185,17 +191,17 @@ func TransformPlanOp(op types.PlanOperator, f PlanOpTransformFunc) (types.PlanOp
sameChildren := true
if len(newChildren) > 0 {
sameChildren = false
op, err = op.WithChildren(newChildren...)
thisOperator, err = thisOperator.WithChildren(newChildren...)
if err != nil {
return nil, true, err
}
}
op, sameOperator, err := f(op)
resultOperator, sameOperator, err := f(thisOperator)
if err != nil {
return nil, true, err
}
return op, sameChildren && sameOperator, nil
return resultOperator, sameChildren && sameOperator, nil
}
// ParentContext is a struct that enables transformation functions to include a parent operator
@ -209,6 +215,7 @@ type ParentContextFunc func(c ParentContext) (types.PlanOperator, bool, error)
type ParentSelectorFunc func(c ParentContext) bool
// TransformPlanOpWithParent applies a transformation function to a plan operator in the context that plan operators parent
func TransformPlanOpWithParent(op types.PlanOperator, s ParentSelectorFunc, f ParentContextFunc) (types.PlanOperator, bool, error) {
return planOpWithParentHelper(ParentContext{op, nil, -1}, s, f)
}
@ -252,11 +259,11 @@ func planOpWithParentHelper(c ParentContext, s ParentSelectorFunc, f ParentConte
}
}
operator, sameOperator, err := f(ParentContext{operator, c.Parent, c.ChildCount})
resultOperator, sameOperator, err := f(ParentContext{operator, c.Parent, c.ChildCount})
if err != nil {
return nil, true, err
}
return operator, sameChildren && sameOperator, nil
return resultOperator, sameChildren && sameOperator, nil
}
// ExprWithPlanOpFunc is a function that given an expression and the node
@ -362,11 +369,13 @@ func TransformSinglePlanOpExpressions(op types.PlanOperator, f ExprFunc) (types.
return op, true, nil
}
// TransformExpr applies a transformation function to an expression
// TransformExpr applies a depth first transformation function to an expression
func TransformExpr(expr types.PlanExpression, f ExprFunc) (types.PlanExpression, bool, error) {
thisExpr := expr
children := expr.Children()
if len(children) == 0 {
return f(expr)
return f(thisExpr)
}
var (
@ -392,22 +401,24 @@ func TransformExpr(expr types.PlanExpression, f ExprFunc) (types.PlanExpression,
sameChildren := true
if len(newChildren) > 0 {
sameChildren = false
expr, err = expr.WithChildren(newChildren...)
thisExpr, err = thisExpr.WithChildren(newChildren...)
if err != nil {
return nil, true, err
}
}
expr, sameExpr, err := f(expr)
resultExpr, sameExpr, err := f(thisExpr)
if err != nil {
return nil, true, err
}
return expr, sameChildren && sameExpr, nil
return resultExpr, sameChildren && sameExpr, nil
}
// TransformExprWithPlanOp applies a transformation function to an expression in the context of a plan operator
// TransformExprWithPlanOp applies a depth first transformation function to an expression in the context of a plan operator
func TransformExprWithPlanOp(n types.PlanOperator, e types.PlanExpression, f ExprWithPlanOpFunc) (types.PlanExpression, bool, error) {
children := e.Children()
thisExpr := e
children := thisExpr.Children()
if len(children) == 0 {
return f(n, e)
}
@ -435,15 +446,15 @@ func TransformExprWithPlanOp(n types.PlanOperator, e types.PlanExpression, f Exp
sameChilren := true
if len(newChildren) > 0 {
sameChilren = false
e, err = e.WithChildren(newChildren...)
thisExpr, err = thisExpr.WithChildren(newChildren...)
if err != nil {
return nil, true, err
}
}
e, sameExpr, err := f(n, e)
resultExpr, sameExpr, err := f(n, thisExpr)
if err != nil {
return nil, true, err
}
return e, sameChilren && sameExpr, nil
return resultExpr, sameChilren && sameExpr, nil
}

5
sql3/planner/types/planexpression.go
View file

@ -2,6 +2,7 @@ package types
import (
"context"
"fmt"
"github.com/molecula/featurebase/v3/sql3/parser"
)
@ -24,6 +25,8 @@ const (
// PlanExpression is an expression node for an execution plan
type PlanExpression interface {
fmt.Stringer
// evaluates expression based on current row
Evaluate(currentRow []interface{}) (interface{}, error)
@ -50,6 +53,8 @@ type AggregationBuffer interface {
// Interface to an expression that is a an aggregate
type Aggregable interface {
fmt.Stringer
NewBuffer() (AggregationBuffer, error)
AggType() AggregateFunctionType
AggExpression() PlanExpression

17
sql3/sql_complex_test.go
View file

@ -19,6 +19,23 @@ import (
"github.com/stretchr/testify/assert"
)
func TestPlanner_Misc(t *testing.T) {
d, err := parser.StringToDecimal("12.345678")
if err != nil {
t.Fatal(err)
}
assert.True(t, d.EqualTo(pql.NewDecimal(12345678, 6)))
d = parser.FloatToDecimalWithScale(12.345678, 6)
if err != nil {
t.Fatal(err)
}
assert.True(t, d.EqualTo(pql.NewDecimal(12345678, 6)))
}
func TestPlanner_Show(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()

49
sql3/test/defs/defs_join.go
View file

@ -1,5 +1,7 @@
package defs
import "github.com/molecula/featurebase/v3/pql"
// join tests
var joinTestsUsers = TableTest{
name: "jointestusers",
@ -54,10 +56,49 @@ var joinTests = TableTest{
hdr("", fldTypeDecimal2),
),
ExpRows: rows(
row(int64(1), float64(22.98)),
row(int64(0), float64(3.99)),
row(int64(2), float64(16.98)),
row(int64(3), float64(5.99)),
row(int64(1), pql.NewDecimal(2298, 2)),
row(int64(0), pql.NewDecimal(399, 2)),
row(int64(2), pql.NewDecimal(1698, 2)),
row(int64(3), pql.NewDecimal(599, 2)),
),
Compare: CompareExactOrdered,
},
{
name: "innerjoin-aggregate-groupby-sum-filter",
SQLs: sqls(
"select sum(price) from orders o inner join users u on o.userid = u._id where u.age > 20;",
),
ExpHdrs: hdrs(
hdr("", fldTypeDecimal2),
),
ExpRows: rows(
row(pql.NewDecimal(2696, 2)),
),
Compare: CompareExactOrdered,
},
{
name: "innerjoin-aggregate-groupby-count-distinct-filter",
SQLs: sqls(
"SELECT COUNT(DISTINCT u.name) FROM orders o JOIN users u ON o.userid = u._id WHERE o.price > 10;",
),
ExpHdrs: hdrs(
hdr("", fldTypeInt),
),
ExpRows: rows(
row(int64(2)),
),
Compare: CompareExactOrdered,
},
{
name: "innerjoin-aggregate-groupby-count-filter",
SQLs: sqls(
"SELECT COUNT(u.name) FROM orders o JOIN users u ON o.userid = u._id WHERE o.price > 10;",
),
ExpHdrs: hdrs(
hdr("", fldTypeInt),
),
ExpRows: rows(
row(int64(2)),
),
Compare: CompareExactOrdered,
},

10
sql3/test/defs/defs_unops.go
View file

@ -1,6 +1,10 @@
package defs
import "time"
import (
"time"
"github.com/molecula/featurebase/v3/pql"
)
var unaryOpExprWithInt = TableTest{
Table: tbl(
@ -157,7 +161,7 @@ var unaryOpExprWithDecimal = TableTest{
hdr("", fldTypeDecimal2),
),
ExpRows: rows(
row(float64(-12.34)),
row(pql.NewDecimal(-1234, 2)),
),
Compare: CompareExactUnordered,
},
@ -175,7 +179,7 @@ var unaryOpExprWithDecimal = TableTest{
hdr("", fldTypeDecimal2),
),
ExpRows: rows(
row(float64(12.34)),
row(pql.NewDecimal(1234, 2)),
),
Compare: CompareExactUnordered,
},

3
sql3/test/helpers.go
View file

@ -20,6 +20,9 @@ func MustQueryRows(tb testing.TB, svr *pilosa.Server, q string) ([][]interface{}
return nil, nil, err
}
// get the plan so that code runs during testing
_ = stmt.Plan()
ocolumns := stmt.Schema()
rowIter, err := stmt.Iterator(ctx, nil)