From a6c165dd571114f6fb2d3969afb54ebe08585534 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula <85502298+pokeeffe-molecula@users.noreply.github.com> Date: Mon, 19 Dec 2022 16:34:28 -0600 Subject: [PATCH] Implement DELETE (fb 1557) (#2382) * delete implementation with test coverage * optimize IN expressions; stop linter complaining * fixed some uncovered query cases * skip test in DAX for now (cherry picked from commit 021219935f56ce7f37179778350e68b591199182) --- dax/test/dax/dax_test.go | 7 + sql3/parser/ast.go | 54 +--- sql3/parser/ast_test.go | 2 +- sql3/parser/parser.go | 38 +-- sql3/parser/parser_test.go | 14 +- sql3/parser/walk.go | 56 ++-- sql3/planner/compiledelete.go | 72 +++++ sql3/planner/compileselect.go | 62 ++--- sql3/planner/executionplanner.go | 72 +---- sql3/planner/expression.go | 1 - sql3/planner/expressionanalyzer.go | 132 +++++++-- sql3/planner/expressionpql.go | 114 +++++++- sql3/planner/oppqldelete.go | 175 ++++++++++++ sql3/planner/oppqlfiltereddelete.go | 113 ++++++++ sql3/planner/opprojection.go | 20 +- sql3/planner/opqltruncate.go | 104 +++++++ sql3/planner/opquery.go | 3 - sql3/planner/opsubquery.go | 5 +- sql3/planner/planoptimizer.go | 242 ++++++++++++----- sql3/planner/planwalker.go | 12 +- sql3/planner/types/planexpression.go | 4 +- sql3/test/defs/defs.go | 2 + sql3/test/defs/defs_delete.go | 389 +++++++++++++++++++++++++++ sql3/test/defs/defs_null.go | 95 ++++++- 24 files changed, 1444 insertions(+), 344 deletions(-) create mode 100644 sql3/planner/compiledelete.go create mode 100644 sql3/planner/oppqldelete.go create mode 100644 sql3/planner/oppqlfiltereddelete.go create mode 100644 sql3/planner/opqltruncate.go create mode 100644 sql3/test/defs/defs_delete.go diff --git a/dax/test/dax/dax_test.go b/dax/test/dax/dax_test.go index cd1b0ce5b..a5f026412 100644 --- a/dax/test/dax/dax_test.go +++ b/dax/test/dax/dax_test.go @@ -111,6 +111,7 @@ func TestDAXIntegration(t *testing.T) { "percentile_test/test-6", // related to TODO in orchestrator.executePercentile "alterTable/alterTableBadTable", // looks like table does not exist is a different error in DAX "top-tests/test-1", // don't know why this is failing at all + "delete_tests", } doSkip := func(name string) bool { @@ -134,12 +135,18 @@ func TestDAXIntegration(t *testing.T) { PQLTests: make([]defs.PQLTest, 0), } for j, sqltest := range test.SQLTests { + if doSkip(test.Name(i)) { + continue + } if doSkip(test.Name(i) + "/" + sqltest.Name(j)) { continue } tt.SQLTests = append(tt.SQLTests, sqltest) } for j, pqltest := range test.PQLTests { + if doSkip(test.Name(i)) { + continue + } if doSkip(test.Name(i) + "/" + pqltest.Name(j)) { continue } diff --git a/sql3/parser/ast.go b/sql3/parser/ast.go index 5d2b65f30..0c90d7c78 100644 --- a/sql3/parser/ast.go +++ b/sql3/parser/ast.go @@ -195,7 +195,7 @@ func StatementSource(stmt Statement) Source { case *UpdateStatement: return stmt.Table case *DeleteStatement: - return stmt.Table + return stmt.Source default: return nil } @@ -3151,23 +3151,14 @@ func (s *UpdateStatement) String() string { } type DeleteStatement struct { - WithClause *WithClause // clause containing CTEs - Delete Pos // position of UPDATE keyword - From Pos // position of FROM keyword - Table *QualifiedTableName // table name + // WithClause *WithClause // clause containing CTEs + Delete Pos // position of UPDATE keyword + From Pos // position of FROM keyword + TableName *QualifiedTableName // the name of the table we are deleting from + Source Source // source for the delete Where Pos // position of WHERE keyword WhereExpr Expr // conditional expression - - Order Pos // position of ORDER keyword - OrderBy Pos // position of BY keyword after ORDER - OrderingTerms []*OrderingTerm // terms of ORDER BY clause - - Limit Pos // position of LIMIT keyword - LimitExpr Expr // limit expression - Offset Pos // position of OFFSET keyword - OffsetComma Pos // position of COMMA (instead of OFFSET) - OffsetExpr Expr // offset expression } // Clone returns a deep copy of s. @@ -3176,47 +3167,20 @@ func (s *DeleteStatement) Clone() *DeleteStatement { return nil } other := *s - other.WithClause = s.WithClause.Clone() - other.Table = s.Table.Clone() + //other.WithClause = s.WithClause.Clone() + other.Source = CloneSource(s.Source) other.WhereExpr = CloneExpr(s.WhereExpr) - other.OrderingTerms = cloneOrderingTerms(s.OrderingTerms) - other.LimitExpr = CloneExpr(s.LimitExpr) - other.OffsetExpr = CloneExpr(s.OffsetExpr) return &other } // String returns the string representation of the clause. func (s *DeleteStatement) String() string { var buf bytes.Buffer - if s.WithClause != nil { - buf.WriteString(s.WithClause.String()) - buf.WriteString(" ") - } - fmt.Fprintf(&buf, "DELETE FROM %s", s.Table.String()) + fmt.Fprintf(&buf, "DELETE FROM %s", s.TableName.String()) if s.WhereExpr != nil { fmt.Fprintf(&buf, " WHERE %s", s.WhereExpr.String()) } - - // Write ORDER BY. - if len(s.OrderingTerms) != 0 { - buf.WriteString(" ORDER BY ") - for i, term := range s.OrderingTerms { - if i != 0 { - buf.WriteString(", ") - } - buf.WriteString(term.String()) - } - } - - // Write LIMIT/OFFSET. - if s.LimitExpr != nil { - fmt.Fprintf(&buf, " LIMIT %s", s.LimitExpr.String()) - if s.OffsetExpr != nil { - fmt.Fprintf(&buf, " OFFSET %s", s.OffsetExpr.String()) - } - } - return buf.String() } diff --git a/sql3/parser/ast_test.go b/sql3/parser/ast_test.go index e03ad6aa5..092b21e6c 100644 --- a/sql3/parser/ast_test.go +++ b/sql3/parser/ast_test.go @@ -415,7 +415,7 @@ func TestCreateViewStatement_String(t *testing.T) { func TestDeleteStatement_String(t *testing.T) { AssertStatementStringer(t, &parser.DeleteStatement{ - Table: &parser.QualifiedTableName{Name: &parser.Ident{Name: "tbl"}, Alias: &parser.Ident{Name: "tbl2"}}, + TableName: &parser.QualifiedTableName{Name: &parser.Ident{Name: "tbl"}, Alias: &parser.Ident{Name: "tbl2"}}, }, `DELETE FROM tbl AS tbl2`) // AssertStatementStringer(t, &sql.DeleteStatement{ diff --git a/sql3/parser/parser.go b/sql3/parser/parser.go index deba6e569..3f19647c8 100644 --- a/sql3/parser/parser.go +++ b/sql3/parser/parser.go @@ -122,7 +122,7 @@ func (p *Parser) parseNonExplainStatement() (Statement, error) { case UPDATE: return p.parseUpdateStatement(nil) case DELETE: - return p.parseDeleteStatement(nil) + return p.parseDeleteStatement() // case WITH: // return p.parseWithStatement() case SHOW: @@ -1883,11 +1883,11 @@ func (p *Parser) parseUpdateStatement(withClause *WithClause) (_ *UpdateStatemen return &stmt, nil } -func (p *Parser) parseDeleteStatement(withClause *WithClause) (_ *DeleteStatement, err error) { +func (p *Parser) parseDeleteStatement( /*withClause *WithClause*/ ) (_ *DeleteStatement, err error) { assert(p.peek() == DELETE) var stmt DeleteStatement - stmt.WithClause = withClause + //stmt.WithClause = withClause // Parse "DELETE FROM tbl" stmt.Delete, _, _ = p.scan() @@ -1900,12 +1900,15 @@ func (p *Parser) parseDeleteStatement(withClause *WithClause) (_ *DeleteStatemen if err != nil { return &stmt, err } - stmt.Table, err = p.parseQualifiedTableName(ident) + tableName, err := p.parseQualifiedTableName(ident) if err != nil { return &stmt, err } + stmt.Source = tableName + // keep the table name too + stmt.TableName = tableName.Clone() - // Parse WHERE clause. + // parse WHERE clause. if p.peek() == WHERE { stmt.Where, _, _ = p.scan() if stmt.WhereExpr, err = p.ParseExpr(); err != nil { @@ -1913,31 +1916,6 @@ func (p *Parser) parseDeleteStatement(withClause *WithClause) (_ *DeleteStatemen } } - // Parse ORDER BY clause. This differs from the SELECT parsing in that - // if an ORDER BY is specified then the LIMIT is required. - if p.peek() == ORDER { - if p.peek() == ORDER { - stmt.Order, _, _ = p.scan() - if p.peek() != BY { - return &stmt, p.errorExpected(p.pos, p.tok, "BY") - } - stmt.OrderBy, _, _ = p.scan() - - for { - term, err := p.parseOrderingTerm() - if err != nil { - return &stmt, err - } - stmt.OrderingTerms = append(stmt.OrderingTerms, term) - - if p.peek() != COMMA { - break - } - p.scan() - } - } - } - return &stmt, nil } diff --git a/sql3/parser/parser_test.go b/sql3/parser/parser_test.go index e08e21bed..74ecaf725 100644 --- a/sql3/parser/parser_test.go +++ b/sql3/parser/parser_test.go @@ -2815,14 +2815,20 @@ func TestParser_ParseStatement(t *testing.T) { AssertParseStatement(t, `DELETE FROM tbl`, &parser.DeleteStatement{ Delete: pos(0), From: pos(7), - Table: &parser.QualifiedTableName{ + TableName: &parser.QualifiedTableName{ + Name: &parser.Ident{NamePos: pos(12), Name: "tbl"}, + }, + Source: &parser.QualifiedTableName{ Name: &parser.Ident{NamePos: pos(12), Name: "tbl"}, }, }) AssertParseStatement(t, `DELETE FROM tbl WHERE x = 1`, &parser.DeleteStatement{ Delete: pos(0), From: pos(7), - Table: &parser.QualifiedTableName{ + TableName: &parser.QualifiedTableName{ + Name: &parser.Ident{NamePos: pos(12), Name: "tbl"}, + }, + Source: &parser.QualifiedTableName{ Name: &parser.Ident{NamePos: pos(12), Name: "tbl"}, }, Where: pos(16), @@ -2902,8 +2908,8 @@ func TestParser_ParseStatement(t *testing.T) { AssertParseStatementError(t, `DELETE`, `1:6: expected FROM, found 'EOF'`) AssertParseStatementError(t, `DELETE FROM`, `1:11: expected table name, found 'EOF'`) AssertParseStatementError(t, `DELETE FROM tbl WHERE`, `1:21: expected expression, found 'EOF'`) - AssertParseStatementError(t, `DELETE FROM tbl ORDER `, `1:22: expected BY, found 'EOF'`) - AssertParseStatementError(t, `DELETE FROM tbl ORDER BY`, `1:24: expected expression, found 'EOF'`) + //AssertParseStatementError(t, `DELETE FROM tbl ORDER `, `1:22: expected BY, found 'EOF'`) + //AssertParseStatementError(t, `DELETE FROM tbl ORDER BY`, `1:24: expected expression, found 'EOF'`) //AssertParseStatementError(t, `DELETE FROM tbl ORDER BY x`, `1:26: expected LIMIT, found 'EOF'`) //AssertParseStatementError(t, `DELETE FROM tbl LIMIT`, `1:21: expected expression, found 'EOF'`) //AssertParseStatementError(t, `DELETE FROM tbl LIMIT 1,`, `1:24: expected expression, found 'EOF'`) diff --git a/sql3/parser/walk.go b/sql3/parser/walk.go index 1607fd10c..2453a5e8b 100644 --- a/sql3/parser/walk.go +++ b/sql3/parser/walk.go @@ -352,42 +352,42 @@ func walk(v Visitor, node Node) (_ Node, err error) { } case *DeleteStatement: - if n.WithClause != nil { - if clause, err := walk(v, n.WithClause); err != nil { - return node, err - } else if clause != nil { - n.WithClause = clause.(*WithClause) - } else { - n.WithClause = nil - } - } - if n.Table != nil { - if tbl, err := walk(v, n.Table); err != nil { + // if n.WithClause != nil { + // if clause, err := walk(v, n.WithClause); err != nil { + // return node, err + // } else if clause != nil { + // n.WithClause = clause.(*WithClause) + // } else { + // n.WithClause = nil + // } + // } + if n.Source != nil { + if tbl, err := walk(v, n.Source); err != nil { return node, err } else if tbl != nil { - n.Table = tbl.(*QualifiedTableName) + n.Source = tbl.(*QualifiedTableName) } else { - n.Table = nil + n.Source = nil } } if err := walkExpr(v, &n.WhereExpr); err != nil { return node, err } - for i := range n.OrderingTerms { - if term, err := walk(v, n.OrderingTerms[i]); err != nil { - return node, err - } else if term != nil { - n.OrderingTerms[i] = term.(*OrderingTerm) - } else { - n.OrderingTerms[i] = nil - } - } - if err := walkExpr(v, &n.LimitExpr); err != nil { - return node, err - } - if err := walkExpr(v, &n.OffsetExpr); err != nil { - return node, err - } + // for i := range n.OrderingTerms { + // if term, err := walk(v, n.OrderingTerms[i]); err != nil { + // return node, err + // } else if term != nil { + // n.OrderingTerms[i] = term.(*OrderingTerm) + // } else { + // n.OrderingTerms[i] = nil + // } + // } + // if err := walkExpr(v, &n.LimitExpr); err != nil { + // return node, err + // } + // if err := walkExpr(v, &n.OffsetExpr); err != nil { + // return node, err + // } case *PrimaryKeyConstraint: if err := walkIdent(v, &n.Name); err != nil { diff --git a/sql3/planner/compiledelete.go b/sql3/planner/compiledelete.go new file mode 100644 index 000000000..2faabeef0 --- /dev/null +++ b/sql3/planner/compiledelete.go @@ -0,0 +1,72 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "github.com/molecula/featurebase/v3/sql3/parser" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +// compileDeleteStatement compiles a parser.DeleteStatment AST into a PlanOperator +func (p *ExecutionPlanner) compileDeleteStatement(stmt *parser.DeleteStatement) (types.PlanOperator, error) { + query := NewPlanOpQuery(p, NewPlanOpNullTable(), p.sql) + + tableName := parser.IdentName(stmt.TableName.Name) + + // source expression + source, err := p.compileSource(query, stmt.Source) + if err != nil { + return nil, err + } + + // handle the where clause + where, err := p.compileExpr(stmt.WhereExpr) + if err != nil { + return nil, err + } + + _, sourceIsScan := source.(*PlanOpPQLTableScan) + + // no where clause and source is a scan so it's a truncate + if where == nil && sourceIsScan { + delOp := NewPlanOpPQLTruncateTable(p, string(tableName)) + + children := []types.PlanOperator{ + delOp, + } + return query.WithChildren(children...) + } + + var delOp types.PlanOperator + + // if we did have a where, insert the filter op + if where != nil { + delOp = NewPlanOpPQLConstRowDelete(p, string(tableName), NewPlanOpFilter(p, where, source)) + } else { + delOp = NewPlanOpPQLConstRowDelete(p, string(tableName), source) + } + + children := []types.PlanOperator{ + delOp, + } + return query.WithChildren(children...) +} + +func (p *ExecutionPlanner) analyzeDeleteStatement(stmt *parser.DeleteStatement) error { + + err := p.analyzeSource(stmt.Source, stmt) + if err != nil { + return err + } + + // if we have a where clause, check that + if stmt.WhereExpr != nil { + expr, err := p.analyzeExpression(stmt.WhereExpr, stmt) + if err != nil { + return err + } + stmt.WhereExpr = expr + } + + return nil +} diff --git a/sql3/planner/compileselect.go b/sql3/planner/compileselect.go index 23906045b..f2fca77c7 100644 --- a/sql3/planner/compileselect.go +++ b/sql3/planner/compileselect.go @@ -17,7 +17,6 @@ import ( // compileSelectStatment compiles a parser.SelectStatment AST into a PlanOperator func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, isSubquery bool) (types.PlanOperator, error) { query := NewPlanOpQuery(p, NewPlanOpNullTable(), p.sql) - p.scopeStack.push(query) aggregates := make([]types.PlanExpression, 0) @@ -59,7 +58,7 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, } // source expression - source, err := p.compileSelectSource(query, stmt.Source) + source, err := p.compileSource(query, stmt.Source) if err != nil { return nil, err } @@ -224,9 +223,6 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, compiledOp = NewPlanOpTop(topExpr, compiledOp) } - // pop the scope - _ = p.scopeStack.pop() - // if it is a subquery, don't wrap in a PlanOpQuery if isSubquery { return compiledOp, nil @@ -263,7 +259,7 @@ func (p *ExecutionPlanner) gatherExprAggregates(expr types.PlanExpression, aggre return result } -func (p *ExecutionPlanner) compileSelectSource(scope *PlanOpQuery, source parser.Source) (types.PlanOperator, error) { +func (p *ExecutionPlanner) compileSource(scope *PlanOpQuery, source parser.Source) (types.PlanOperator, error) { if source == nil { return NewPlanOpNullTable(), nil } @@ -289,11 +285,11 @@ func (p *ExecutionPlanner) compileSelectSource(scope *PlanOpQuery, source parser } } - topOp, err := p.compileSelectSource(scope, sourceExpr.X) + topOp, err := p.compileSource(scope, sourceExpr.X) if err != nil { return nil, err } - bottomOp, err := p.compileSelectSource(scope, sourceExpr.Y) + bottomOp, err := p.compileSource(scope, sourceExpr.Y) if err != nil { return nil, err } @@ -313,22 +309,14 @@ func (p *ExecutionPlanner) compileSelectSource(scope *PlanOpQuery, source parser return NewPlanOpSystemTable(p, st), nil } - // get all the qualified refs that refer to this table + // get all the columns for this table - we will eliminate unused ones + // later on in the optimizer extractColumns := make([]string, 0) - for _, r := range scope.referenceList { - if sourceExpr.MatchesTablenameOrAlias(r.tableName) { - found := false - for _, c := range extractColumns { - if strings.EqualFold(c, r.columnName) { - found = true - break - } - } - if !found { - extractColumns = append(extractColumns, r.columnName) - } - } + + for _, oc := range sourceExpr.OutputColumns { + extractColumns = append(extractColumns, oc.ColumnName) } + if sourceExpr.Alias != nil { aliasName := parser.IdentName(sourceExpr.Alias) @@ -352,14 +340,14 @@ func (p *ExecutionPlanner) compileSelectSource(scope *PlanOpQuery, source parser case *parser.ParenSource: if sourceExpr.Alias != nil { aliasName := parser.IdentName(sourceExpr.Alias) - op, err := p.compileSelectSource(scope, sourceExpr.X) + op, err := p.compileSource(scope, sourceExpr.X) if err != nil { return nil, err } return NewPlanOpRelAlias(aliasName, op), nil } - return p.compileSelectSource(scope, sourceExpr.X) + return p.compileSource(scope, sourceExpr.X) case *parser.SelectStatement: subQuery, err := p.compileSelectStatement(sourceExpr, true) @@ -465,7 +453,7 @@ func (p *ExecutionPlanner) analyzeSource(source parser.Source, scope parser.Stat return nil case *parser.SelectStatement: - err := p.analyzeSelectStatement(source) + _, err := p.analyzeSelectStatement(source) if err != nil { return err } @@ -476,21 +464,21 @@ func (p *ExecutionPlanner) analyzeSource(source parser.Source, scope parser.Stat } } -func (p *ExecutionPlanner) analyzeSelectStatement(stmt *parser.SelectStatement) error { +func (p *ExecutionPlanner) analyzeSelectStatement(stmt *parser.SelectStatement) (parser.Expr, error) { // analyze source first - needed for name resolution err := p.analyzeSource(stmt.Source, stmt) if err != nil { - return err + return nil, err } if err := p.analyzeSelectStatementWildcards(stmt); err != nil { - return err + return nil, err } for _, col := range stmt.Columns { expr, err := p.analyzeExpression(col.Expr, stmt) if err != nil { - return err + return nil, err } if expr != nil { col.Expr = expr @@ -499,31 +487,31 @@ func (p *ExecutionPlanner) analyzeSelectStatement(stmt *parser.SelectStatement) expr, err := p.analyzeExpression(stmt.TopExpr, stmt) if err != nil { - return err + return nil, err } if expr != nil { if !(expr.IsLiteral() && typeIsInteger(expr.DataType())) { - return sql3.NewErrIntegerLiteral(stmt.TopExpr.Pos().Line, stmt.TopExpr.Pos().Column) + return nil, sql3.NewErrIntegerLiteral(stmt.TopExpr.Pos().Line, stmt.TopExpr.Pos().Column) } stmt.TopExpr = expr } expr, err = p.analyzeExpression(stmt.HavingExpr, stmt) if err != nil { - return err + return nil, err } stmt.HavingExpr = expr expr, err = p.analyzeExpression(stmt.WhereExpr, stmt) if err != nil { - return err + return nil, err } stmt.WhereExpr = expr for i, g := range stmt.GroupByExprs { expr, err = p.analyzeExpression(g, stmt) if err != nil { - return err + return nil, err } if expr != nil { stmt.GroupByExprs[i] = expr @@ -532,7 +520,7 @@ func (p *ExecutionPlanner) analyzeSelectStatement(stmt *parser.SelectStatement) expr, err = p.analyzeExpression(stmt.HavingExpr, stmt) if err != nil { - return err + return nil, err } if expr != nil { stmt.HavingExpr = expr @@ -541,12 +529,12 @@ func (p *ExecutionPlanner) analyzeSelectStatement(stmt *parser.SelectStatement) for _, term := range stmt.OrderingTerms { expr, err := p.analyzeOrderingTermExpression(term.X, stmt) if err != nil { - return err + return nil, err } term.X = expr } - return nil + return stmt, nil } func (p *ExecutionPlanner) analyzeSelectStatementWildcards(stmt *parser.SelectStatement) error { diff --git a/sql3/planner/executionplanner.go b/sql3/planner/executionplanner.go index 91355ba09..5f7f59106 100644 --- a/sql3/planner/executionplanner.go +++ b/sql3/planner/executionplanner.go @@ -12,14 +12,6 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) -// PlannerScope holds scope for the planner -// there is a stack of these in the ExecutionPlanner and some corresponding push/pop functions -// this allows us to do scoped operations without passing stuff down into -// every function -type PlannerScope struct { - scope types.PlanOperator -} - // ExecutionPlanner compiles SQL text into a query plan type ExecutionPlanner struct { executor pilosa.Executor @@ -29,7 +21,6 @@ type ExecutionPlanner struct { importer pilosa.Importer logger logger.Logger sql string - scopeStack *scopeStack } func NewExecutionPlanner(executor pilosa.Executor, schemaAPI pilosa.SchemaAPI, systemAPI pilosa.SystemAPI, systemLayerAPI pilosa.SystemLayerAPI, importer pilosa.Importer, logger logger.Logger, sql string) *ExecutionPlanner { @@ -41,7 +32,6 @@ func NewExecutionPlanner(executor pilosa.Executor, schemaAPI pilosa.SchemaAPI, s importer: importer, logger: logger, sql: sql, - scopeStack: newScopeStack(), } } @@ -77,6 +67,8 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen rootOperator, err = p.compileInsertStatement(stmt) case *parser.BulkInsertStatement: rootOperator, err = p.compileBulkInsertStatement(stmt) + case *parser.DeleteStatement: + rootOperator, err = p.compileDeleteStatement(stmt) default: return nil, sql3.NewErrInternalf("cannot plan statement: %T", stmt) } @@ -90,7 +82,8 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen func (p *ExecutionPlanner) analyzePlan(stmt parser.Statement) error { switch stmt := stmt.(type) { case *parser.SelectStatement: - return p.analyzeSelectStatement(stmt) + _, err := p.analyzeSelectStatement(stmt) + return err case *parser.ShowTablesStatement: return nil case *parser.ShowColumnsStatement: @@ -107,6 +100,8 @@ func (p *ExecutionPlanner) analyzePlan(stmt parser.Statement) error { return p.analyzeInsertStatement(stmt) case *parser.BulkInsertStatement: return p.analyzeBulkInsertStatement(stmt) + case *parser.DeleteStatement: + return p.analyzeDeleteStatement(stmt) default: return sql3.NewErrInternalf("cannot analyze statement: %T", stmt) } @@ -125,58 +120,3 @@ const ( func (p *ExecutionPlanner) checkAccess(ctx context.Context, objectName string, _ accessType) error { return nil } - -// addReference is a convenience function that allows the planner to keep track -// of references so we can use them during optimization. -func (p *ExecutionPlanner) addReference(ref *qualifiedRefPlanExpression) error { - table := p.scopeStack.read() - if table == nil { - return sql3.NewErrInternalf("unexpected symbol table state") - } - - switch s := table.scope.(type) { - case *PlanOpQuery: - s.referenceList = append(s.referenceList, ref) - } - return nil -} - -// scopeStack is a stack of PlannerScope with the usual push/pop methods. -type scopeStack struct { - st []*PlannerScope -} - -// newScopeStack returns a scope stack initialized with zero elements on the -// stack. -func newScopeStack() *scopeStack { - return &scopeStack{ - st: make([]*PlannerScope, 0), - } -} - -// push adds the provided PlanOperator (as the scope of a PlannerScope) to the -// scope stack. -func (ss *scopeStack) push(scope types.PlanOperator) { - ss.st = append(ss.st, &PlannerScope{ - scope: scope, - }) -} - -// pop removes (and returns) the last scope pushed to the stack. -func (ss *scopeStack) pop() *PlannerScope { - if len(ss.st) == 0 { - return nil - } - ret := ss.st[len(ss.st)-1] - ss.st = ss.st[:len(ss.st)-1] - return ret -} - -// read returns the last scope pushed to the stack, but unlike pop, it does not -// remove it. -func (ss *scopeStack) read() *PlannerScope { - if len(ss.st) == 0 { - return nil - } - return ss.st[len(ss.st)-1] -} diff --git a/sql3/planner/expression.go b/sql3/planner/expression.go index 584a946f7..d429a3842 100644 --- a/sql3/planner/expression.go +++ b/sql3/planner/expression.go @@ -2525,7 +2525,6 @@ func (p *ExecutionPlanner) compileExpr(expr parser.Expr) (_ types.PlanExpression case *parser.QualifiedRef: ref := newQualifiedRefPlanExpression(parser.IdentName(expr.Table), parser.IdentName(expr.Column), expr.ColumnIndex, expr.DataType()) - p.addReference(ref) return ref, nil case *parser.Range: diff --git a/sql3/planner/expressionanalyzer.go b/sql3/planner/expressionanalyzer.go index 1b1c2bac5..aedd6b446 100644 --- a/sql3/planner/expressionanalyzer.go +++ b/sql3/planner/expressionanalyzer.go @@ -56,7 +56,6 @@ func (p *ExecutionPlanner) analyzeExpression(expr parser.Expr, scope parser.Stat case *parser.Ident: switch sc := scope.(type) { case *parser.SelectStatement: - // turn *parser.Ident into *parser.QualifiedRef if sc.Source == nil { return nil, sql3.NewErrColumnNotFound(e.NamePos.Line, e.NamePos.Column, e.Name) } @@ -69,6 +68,33 @@ func (p *ExecutionPlanner) analyzeExpression(expr parser.Expr, scope parser.Stat return nil, sql3.NewErrColumnNotFound(e.NamePos.Line, e.NamePos.Column, e.Name) } + // now turn *parser.Ident into *parser.QualifiedRef + ident := &parser.QualifiedRef{ + Table: &parser.Ident{ + Name: oc.TableName, + NamePos: e.NamePos, + }, + Column: &parser.Ident{ + Name: oc.ColumnName, + NamePos: e.NamePos, + }, + ColumnIndex: oc.ColumnIndex, + } + return p.analyzeExpression(ident, scope) + + case *parser.InsertStatement: + return nil, sql3.NewErrColumnNotFound(e.NamePos.Line, e.NamePos.Column, e.Name) + + case *parser.DeleteStatement: + + // go find the first ident in the source that matches + oc, err := sc.Source.OutputColumnNamed(e.Name) + if err != nil { + return nil, err + } else if oc == nil { + return nil, sql3.NewErrColumnNotFound(e.NamePos.Line, e.NamePos.Column, e.Name) + } + ident := &parser.QualifiedRef{ Table: &parser.Ident{ Name: oc.TableName, @@ -82,9 +108,6 @@ func (p *ExecutionPlanner) analyzeExpression(expr parser.Expr, scope parser.Stat } return p.analyzeExpression(ident, scope) - case *parser.InsertStatement: - return nil, sql3.NewErrColumnNotFound(e.NamePos.Line, e.NamePos.Column, e.Name) - default: return nil, sql3.NewErrInternalf("unhandled scope type '%T'", sc) } @@ -224,6 +247,19 @@ func (p *ExecutionPlanner) analyzeExpression(expr parser.Expr, scope parser.Stat return nil, sql3.NewErrColumnNotFound(e.Column.NamePos.Line, e.Column.NamePos.Column, e.Column.Name) } + case *parser.DeleteStatement: + oc, err := sc.Source.OutputColumnNamed(e.Column.Name) + if err != nil { + return nil, err + } + if oc != nil { + e.RefDataType = oc.Datatype + e.ColumnIndex = oc.ColumnIndex + return e, nil + + } + return nil, sql3.NewErrColumnNotFound(e.Column.NamePos.Line, e.Column.NamePos.Column, e.Column.Name) + default: return nil, sql3.NewErrInternalf("unhandled scope type '%T'", sc) } @@ -298,7 +334,7 @@ func (p *ExecutionPlanner) analyzeExpression(expr parser.Expr, scope parser.Stat return p.analyzeUnaryExpression(e, scope) case *parser.SelectStatement: - err := p.analyzeSelectStatement(e) + selExpr, err := p.analyzeSelectStatement(e) if err != nil { return nil, err } @@ -306,7 +342,7 @@ func (p *ExecutionPlanner) analyzeExpression(expr parser.Expr, scope parser.Stat if len(e.Columns) > 1 { return nil, sql3.NewErrInternalf("subquery must return only one column") } - return e, nil + return selExpr, nil default: return nil, sql3.NewErrInternalf("unexpected SQL expression type: %T", expr) @@ -368,6 +404,18 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop } expr.Y = y + // check nil for either of these expressions after they were ananlyzed, they may have been eliminated + // in which case we return the remaining one or nil if both have been eliminated + if x == nil && y == nil { + return nil, nil + } + if x == nil { + return y, nil + } + if y == nil { + return x, nil + } + //handle operator switch op := expr.Op; op { @@ -576,45 +624,71 @@ func (p *ExecutionPlanner) analyzeBinaryExpression(expr *parser.BinaryExpr, scop if !typesAreComparable(x.DataType(), sel.Columns[0].Expr.DataType()) { return nil, sql3.NewErrTypesAreNotEquatable(x.Pos().Line, x.Pos().Column, x.DataType().TypeDescription(), ex.DataType().TypeDescription()) } - //need to turn this into an inner join - selStmt, ok := scope.(*parser.SelectStatement) - if !ok { - return nil, sql3.NewErrInternalf("unexpected scope type '%T'", scope) - } - operator := &parser.JoinOperator{ Inner: expr.OpPos, } constraint := &parser.OnConstraint{ X: &parser.BinaryExpr{ - X: expr.X, - Op: parser.EQ, - Y: sel.Columns[0].Expr, + X: expr.X, + Op: parser.EQ, + Y: sel.Columns[0].Expr, + ResultDataType: parser.NewDataTypeBool(), }, } - if lhs, ok := selStmt.Source.(*parser.JoinClause); ok { - selStmt.Source = &parser.JoinClause{ - X: lhs.X, - Operator: lhs.Operator, - Y: &parser.JoinClause{ - X: lhs.Y, + switch scopeStmt := scope.(type) { + case *parser.SelectStatement: + + if lhs, ok := scopeStmt.Source.(*parser.JoinClause); ok { + scopeStmt.Source = &parser.JoinClause{ + X: lhs.X, + Operator: lhs.Operator, + Y: &parser.JoinClause{ + X: lhs.Y, + Operator: operator, + Y: sel, + Constraint: constraint, + }, + Constraint: lhs.Constraint, + } + } else { + scopeStmt.Source = &parser.JoinClause{ + X: scopeStmt.Source, Operator: operator, Y: sel, Constraint: constraint, - }, - Constraint: lhs.Constraint, + } } - } else { - selStmt.Source = &parser.JoinClause{ - X: selStmt.Source, - Operator: operator, - Y: sel, - Constraint: constraint, + + case *parser.DeleteStatement: + if lhs, ok := scopeStmt.Source.(*parser.JoinClause); ok { + scopeStmt.Source = &parser.JoinClause{ + X: lhs.X, + Operator: lhs.Operator, + Y: &parser.JoinClause{ + X: lhs.Y, + Operator: operator, + Y: sel, + Constraint: constraint, + }, + Constraint: lhs.Constraint, + } + } else { + scopeStmt.Source = &parser.JoinClause{ + X: scopeStmt.Source, + Operator: operator, + Y: sel, + Constraint: constraint, + } } + + default: + return nil, sql3.NewErrInternalf("unexpected scope type '%T'", scope) } + // we are eliminating this expression, since we moved it into the source, so + // return nil return nil, nil } diff --git a/sql3/planner/expressionpql.go b/sql3/planner/expressionpql.go index b425aacfb..4af10a11a 100644 --- a/sql3/planner/expressionpql.go +++ b/sql3/planner/expressionpql.go @@ -4,6 +4,7 @@ package planner import ( "context" + "strconv" "strings" "github.com/featurebasedb/featurebase/v3/pql" @@ -98,6 +99,60 @@ func (p *ExecutionPlanner) generatePQLCallFromExpr(ctx context.Context, expr typ return nil, sql3.NewErrInternalf("unsupported scalar function '%s'", expr.name) } + case *inOpPlanExpression: + // lhs will be qualified ref + lhs, ok := expr.lhs.(*qualifiedRefPlanExpression) + if !ok { + return nil, sql3.NewErrInternalf("unexpected lhs %T", expr.lhs) + } + + // rhs is expression list - need to convert to a big OR + + list, ok := expr.rhs.(*exprListPlanExpression) + if !ok { + return nil, sql3.NewErrInternalf("unexpected argument type '%T'", expr.rhs) + } + + // if it is the _id column, we can use ConstRow with a list + if strings.EqualFold(lhs.columnName, "_id") { + values := make([]interface{}, len(list.exprs)) + for i, m := range list.exprs { + pqlValue, err := planExprToValue(m) + if err != nil { + return nil, err + } + values[i] = pqlValue + } + call := &pql.Call{ + Name: "ConstRow", + Args: map[string]interface{}{ + "columns": values, + }, + Type: pql.PrecallGlobal, + } + return call, nil + } + // otherwise, OR them all + call := &pql.Call{ + Name: "Union", + Children: []*pql.Call{}, + } + + for _, m := range list.exprs { + pqlValue, err := planExprToValue(m) + if err != nil { + return nil, err + } + rc := &pql.Call{ + Name: "Row", + Args: map[string]interface{}{ + lhs.columnName: pqlValue, + }, + } + call.Children = append(call.Children, rc) + } + return call, nil + default: return nil, sql3.NewErrInternalf("unexpected expression type: %T", expr) } @@ -153,13 +208,26 @@ func (p *ExecutionPlanner) generatePQLCallFromBinaryExpr(ctx context.Context, ex }, nil case *parser.DataTypeID: + // TODO (pok) range queries on _id are not supported if strings.EqualFold(lhs.columnName, "_id") { - return &pql.Call{ + cr := &pql.Call{ Name: "ConstRow", Args: map[string]interface{}{ "columns": []interface{}{pqlValue}, }, Type: pql.PrecallGlobal, + } + // TODO (pok) when we fix FB-1828 (https://molecula.atlassian.net/browse/FB-1828) + // we can remove this - ConstRow returns a ghost record, thus to eliminate + // we interset with All + return &pql.Call{ + Name: "Intersect", + Children: []*pql.Call{ + { + Name: "All", + }, + cr, + }, }, nil } return &pql.Call{ @@ -170,13 +238,26 @@ func (p *ExecutionPlanner) generatePQLCallFromBinaryExpr(ctx context.Context, ex }, nil case *parser.DataTypeString: + // TODO (pok) range queries on _id are not supported if strings.EqualFold(lhs.columnName, "_id") { - return &pql.Call{ + cr := &pql.Call{ Name: "ConstRow", Args: map[string]interface{}{ "columns": []interface{}{pqlValue}, }, Type: pql.PrecallGlobal, + } + // TODO (pok) when we fix FB-1828 (https://molecula.atlassian.net/browse/FB-1828) + // we can remove this - ConstRow returns a ghost record, thus to eliminate + // we interset with All + return &pql.Call{ + Name: "Intersect", + Children: []*pql.Call{ + { + Name: "All", + }, + cr, + }, }, nil } return &pql.Call{ @@ -194,6 +275,27 @@ func (p *ExecutionPlanner) generatePQLCallFromBinaryExpr(ctx context.Context, ex }, }, nil + case *parser.DataTypeBool: + return &pql.Call{ + Name: "Row", + Args: map[string]interface{}{ + lhs.columnName: pqlValue, + }, + }, nil + + case *parser.DataTypeDecimal: + val, ok := pqlValue.(float64) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T", pqlValue) + } + d := pql.FromFloat64(val) + return &pql.Call{ + Name: "Row", + Args: map[string]interface{}{ + lhs.columnName: d, + }, + }, nil + default: return nil, sql3.NewErrInternalf("unsupported type for binary expression: %v (%T)", typ, typ) } @@ -279,6 +381,14 @@ func planExprToValue(expr types.PlanExpression) (interface{}, error) { return expr.value, nil case *dateLiteralPlanExpression: return expr.value, nil + case *boolLiteralPlanExpression: + return expr.value, nil + case *floatLiteralPlanExpression: + f, err := strconv.ParseFloat(expr.value, 64) + if err != nil { + return nil, err + } + return f, nil default: return nil, sql3.NewErrInternalf("cannot convert SQL expression %T to a literal value", expr) } diff --git a/sql3/planner/oppqldelete.go b/sql3/planner/oppqldelete.go new file mode 100644 index 000000000..1c84706ff --- /dev/null +++ b/sql3/planner/oppqldelete.go @@ -0,0 +1,175 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "fmt" + + "github.com/molecula/featurebase/v3/dax" + "github.com/molecula/featurebase/v3/pql" + "github.com/molecula/featurebase/v3/sql3" + "github.com/molecula/featurebase/v3/sql3/parser" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +// PlanOpPQLConstRowDelete plan operator to delete rows from a table based on a single key. +type PlanOpPQLConstRowDelete struct { + planner *ExecutionPlanner + ChildOp types.PlanOperator + tableName string + warnings []string +} + +func NewPlanOpPQLConstRowDelete(p *ExecutionPlanner, tableName string, child types.PlanOperator) *PlanOpPQLConstRowDelete { + return &PlanOpPQLConstRowDelete{ + planner: p, + ChildOp: child, + tableName: tableName, + warnings: make([]string, 0), + } +} + +func (p *PlanOpPQLConstRowDelete) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + ps := make([]string, 0) + for _, e := range p.Schema() { + ps = append(ps, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeDescription())) + } + result["_schema"] = ps + result["child"] = p.ChildOp.Plan() + result["tableName"] = p.tableName + return result +} + +func (p *PlanOpPQLConstRowDelete) String() string { + return "" +} + +func (p *PlanOpPQLConstRowDelete) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpPQLConstRowDelete) Warnings() []string { + return p.warnings +} + +func (p *PlanOpPQLConstRowDelete) Schema() types.Schema { + return types.Schema{} +} + +func (p *PlanOpPQLConstRowDelete) Children() []types.PlanOperator { + return []types.PlanOperator{ + p.ChildOp, + } +} + +func (p *PlanOpPQLConstRowDelete) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + childIter, err := p.ChildOp.Iterator(ctx, row) + if err != nil { + return nil, err + } + + return &constRowDeleteRowIter{ + planner: p.planner, + childIter: childIter, + tableName: p.tableName, + }, nil +} + +func (p *PlanOpPQLConstRowDelete) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + if len(children) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return NewPlanOpPQLConstRowDelete(p.planner, p.tableName, children[0]), nil +} + +func (p *PlanOpPQLConstRowDelete) Expressions() []types.PlanExpression { + // since we have to do const row lookups, we should always reference the _id column in the + // table we are deleting from + + tbl, err := p.planner.schemaAPI.TableByName(context.Background(), dax.TableName(p.tableName)) + if err != nil { + return []types.PlanExpression{} + } + + var colType parser.ExprDataType + if tbl.StringKeys() { + colType = parser.NewDataTypeString() + } else { + colType = parser.NewDataTypeID() + } + + return []types.PlanExpression{ + &qualifiedRefPlanExpression{ + tableName: p.tableName, + columnIndex: 0, + dataType: colType, + columnName: "_id", + }, + } +} + +func (p *PlanOpPQLConstRowDelete) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) { + // just return ourselves + return p, nil +} + +type constRowDeleteRowIter struct { + planner *ExecutionPlanner + childIter types.RowIterator + tableName string +} + +var _ types.RowIterator = (*constRowDeleteRowIter)(nil) + +func (i *constRowDeleteRowIter) Next(ctx context.Context) (types.Row, error) { + var err error + + err = i.planner.checkAccess(ctx, i.tableName, accessTypeWriteData) + if err != nil { + return nil, err + } + + var row []interface{} + row, err = i.childIter.Next(ctx) + if err != nil { + return nil, err + } + keys := make([]interface{}, 0) + for { + keys = append(keys, row[0]) + + row, err = i.childIter.Next(ctx) + if err == types.ErrNoMoreRows { + break + } + if err != nil { + return nil, err + } + } + + if len(keys) > 0 { + // this row should contain the key to delete + cond := &pql.Call{ + Name: "ConstRow", + Args: map[string]interface{}{ + "columns": keys, + }, + Type: pql.PrecallGlobal, + } + call := &pql.Call{Name: "Delete", Children: []*pql.Call{cond}} + + tbl, err := i.planner.schemaAPI.TableByName(ctx, dax.TableName(i.tableName)) + if err != nil { + return nil, sql3.NewErrTableNotFound(0, 0, i.tableName) + } + + _, err = i.planner.executor.Execute(ctx, tbl, &pql.Query{Calls: []*pql.Call{call}}, nil, nil) + if err != nil { + return nil, err + } + } + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/oppqlfiltereddelete.go b/sql3/planner/oppqlfiltereddelete.go new file mode 100644 index 000000000..0e48ff34d --- /dev/null +++ b/sql3/planner/oppqlfiltereddelete.go @@ -0,0 +1,113 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "fmt" + + "github.com/molecula/featurebase/v3/dax" + "github.com/molecula/featurebase/v3/pql" + "github.com/molecula/featurebase/v3/sql3" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +// PlanOpPQLFilteredDelete plan operator to delete rows from a table based on a filter expression. +type PlanOpPQLFilteredDelete struct { + planner *ExecutionPlanner + tableName string + filter types.PlanExpression + warnings []string +} + +func NewPlanOpPQLFilteredDelete(p *ExecutionPlanner, tableName string, filter types.PlanExpression) *PlanOpPQLFilteredDelete { + return &PlanOpPQLFilteredDelete{ + planner: p, + tableName: tableName, + filter: filter, + warnings: make([]string, 0), + } +} + +func (p *PlanOpPQLFilteredDelete) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + ps := make([]string, 0) + for _, e := range p.Schema() { + ps = append(ps, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeDescription())) + } + result["_schema"] = ps + result["filter"] = p.filter.Plan() + result["tableName"] = p.tableName + return result +} + +func (p *PlanOpPQLFilteredDelete) String() string { + return "" +} + +func (p *PlanOpPQLFilteredDelete) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpPQLFilteredDelete) Warnings() []string { + return p.warnings +} + +func (p *PlanOpPQLFilteredDelete) Schema() types.Schema { + return types.Schema{} +} + +func (p *PlanOpPQLFilteredDelete) Children() []types.PlanOperator { + return []types.PlanOperator{} +} + +func (p *PlanOpPQLFilteredDelete) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + return &filteredDeleteRowIter{ + planner: p.planner, + tableName: p.tableName, + filter: p.filter, + }, nil +} + +func (p *PlanOpPQLFilteredDelete) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + if len(children) != 0 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return NewPlanOpPQLFilteredDelete(p.planner, p.tableName, p.filter), nil +} + +type filteredDeleteRowIter struct { + planner *ExecutionPlanner + tableName string + filter types.PlanExpression +} + +var _ types.RowIterator = (*filteredDeleteRowIter)(nil) + +func (i *filteredDeleteRowIter) Next(ctx context.Context) (types.Row, error) { + var err error + + err = i.planner.checkAccess(ctx, i.tableName, accessTypeWriteData) + if err != nil { + return nil, err + } + + cond, err := i.planner.generatePQLCallFromExpr(ctx, i.filter) + if err != nil { + return nil, err + } + call := &pql.Call{Name: "Delete", Children: []*pql.Call{cond}} + + tbl, err := i.planner.schemaAPI.TableByName(ctx, dax.TableName(i.tableName)) + if err != nil { + return nil, sql3.NewErrTableNotFound(0, 0, i.tableName) + } + + _, err = i.planner.executor.Execute(ctx, tbl, &pql.Query{Calls: []*pql.Call{call}}, nil, nil) + if err != nil { + return nil, err + } + + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/opprojection.go b/sql3/planner/opprojection.go index d3c99fd37..32f1bc231 100644 --- a/sql3/planner/opprojection.go +++ b/sql3/planner/opprojection.go @@ -107,22 +107,22 @@ func (p *PlanOpProjection) WithUpdatedExpressions(exprs ...types.PlanExpression) } func ExpressionToColumn(e types.PlanExpression) *types.PlannerColumn { - var name string - if n, ok := e.(types.IdentifiableByName); ok { - name = n.Name() - } else { - name = "" //e.String() - } + name := "" + relationName := "" - var table string - if t, ok := e.(types.IdentifiableByName); ok { - table = t.Name() + switch thisExpr := e.(type) { + case *qualifiedRefPlanExpression: + name = thisExpr.columnName + relationName = thisExpr.tableName + + case *aliasPlanExpression: + name = thisExpr.aliasName } return &types.PlannerColumn{ ColumnName: name, + RelationName: relationName, Type: e.Type(), - RelationName: table, } } diff --git a/sql3/planner/opqltruncate.go b/sql3/planner/opqltruncate.go new file mode 100644 index 000000000..c479bafc4 --- /dev/null +++ b/sql3/planner/opqltruncate.go @@ -0,0 +1,104 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "fmt" + + "github.com/molecula/featurebase/v3/dax" + "github.com/molecula/featurebase/v3/pql" + "github.com/molecula/featurebase/v3/sql3" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +// TODO (pok) we should look at a drop and recreate, or an actual truncate PQL op + +// PlanOpPQLTruncateTable plan operator to delete rows from a table based on a single key. +type PlanOpPQLTruncateTable struct { + planner *ExecutionPlanner + tableName string + warnings []string +} + +func NewPlanOpPQLTruncateTable(p *ExecutionPlanner, tableName string) *PlanOpPQLTruncateTable { + return &PlanOpPQLTruncateTable{ + planner: p, + tableName: tableName, + warnings: make([]string, 0), + } +} + +func (p *PlanOpPQLTruncateTable) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + ps := make([]string, 0) + for _, e := range p.Schema() { + ps = append(ps, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeDescription())) + } + result["_schema"] = ps + result["tableName"] = p.tableName + return result +} + +func (p *PlanOpPQLTruncateTable) String() string { + return "" +} + +func (p *PlanOpPQLTruncateTable) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpPQLTruncateTable) Warnings() []string { + return p.warnings +} + +func (p *PlanOpPQLTruncateTable) Schema() types.Schema { + return types.Schema{} +} + +func (p *PlanOpPQLTruncateTable) Children() []types.PlanOperator { + return []types.PlanOperator{} +} + +func (p *PlanOpPQLTruncateTable) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + return &truncateTableRowIter{ + planner: p.planner, + tableName: p.tableName, + }, nil +} + +func (p *PlanOpPQLTruncateTable) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + return p, nil +} + +type truncateTableRowIter struct { + planner *ExecutionPlanner + tableName string +} + +var _ types.RowIterator = (*truncateTableRowIter)(nil) + +func (i *truncateTableRowIter) Next(ctx context.Context) (types.Row, error) { + err := i.planner.checkAccess(ctx, i.tableName, accessTypeWriteData) + if err != nil { + return nil, err + } + + cond := &pql.Call{ + Name: "All", + } + call := &pql.Call{Name: "Delete", Children: []*pql.Call{cond}} + + tbl, err := i.planner.schemaAPI.TableByName(ctx, dax.TableName(i.tableName)) + if err != nil { + return nil, sql3.NewErrTableNotFound(0, 0, i.tableName) + } + + _, err = i.planner.executor.Execute(ctx, tbl, &pql.Query{Calls: []*pql.Call{call}}, nil, nil) + if err != nil { + return nil, err + } + + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/opquery.go b/sql3/planner/opquery.go index 027718b66..e68ed8e5f 100644 --- a/sql3/planner/opquery.go +++ b/sql3/planner/opquery.go @@ -18,9 +18,6 @@ type PlanOpQuery struct { ChildOp types.PlanOperator - // all the identifiers that are referenced - referenceList []*qualifiedRefPlanExpression - sql string warnings []string } diff --git a/sql3/planner/opsubquery.go b/sql3/planner/opsubquery.go index d2259663c..6425012d4 100644 --- a/sql3/planner/opsubquery.go +++ b/sql3/planner/opsubquery.go @@ -37,7 +37,10 @@ func (p *PlanOpSubquery) Children() []types.PlanOperator { } func (p *PlanOpSubquery) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { - return nil, nil + if len(children) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return NewPlanOpSubquery(children[0]), nil } func (p *PlanOpSubquery) Plan() map[string]interface{} { diff --git a/sql3/planner/planoptimizer.go b/sql3/planner/planoptimizer.go index 874e05194..2e46ea846 100644 --- a/sql3/planner/planoptimizer.go +++ b/sql3/planner/planoptimizer.go @@ -23,12 +23,18 @@ type OptimizerFunc func(context.Context, *ExecutionPlanner, types.PlanOperator, // a list of optimzer rules; order can be important important var optimizerFunctions = []OptimizerFunc{ + // fix expression references for having + removeUnusedExtractColumnReferences, + // fix expression references for having fixHavingReferences, // push down filter predicates as far as possible, pushdownFilters, + // try to use a PlanOpPQLFilteredDelete instead of PlanOpPQLConstRowDelete + tryToReplaceConstRowDeleteWithFilteredDelete, + // if we have a group by that has one TableScanOperator, // try to use a PQL(multi)groupby operator instead tryToReplaceGroupByWithPQLGroupBy, @@ -58,15 +64,26 @@ var optimizerFunctions = []OptimizerFunc{ type OptimizerScope struct { } +func dumpPlan(prefix []string, root types.PlanOperator, suffix string) { + // DEBUG !! + // for _, s := range prefix { + // log.Println(s) + // } + // jplan := root.Plan() + // a, _ := json.MarshalIndent(jplan, "", " ") + // log.Println(string(a)) + // log.Println() + // DEBUG !! +} + // optimizePlan takes a plan from the compiler and executes a series of transforms on it to optimize it func (p *ExecutionPlanner) optimizePlan(ctx context.Context, plan types.PlanOperator) (types.PlanOperator, error) { - // log.Println("================================================================================") - // log.Println("plan pre-optimzation") - // jplan := plan.Plan() - // a, _ := json.MarshalIndent(jplan, "", " ") - // log.Println(string(a)) - // log.Println("--------------------------------------------------------------------------------") + dumpPlan( + []string{"================================================================================", "plan pre-optimzation"}, + plan, + "--------------------------------------------------------------------------------", + ) var err error var result = plan @@ -77,12 +94,11 @@ func (p *ExecutionPlanner) optimizePlan(ctx context.Context, plan types.PlanOper } } - // log.Println("================================================================================") - // log.Println("plan ppst-optimzation") - // jplan = result.Plan() - // a, _ = json.MarshalIndent(jplan, "", " ") - // log.Println(string(a)) - // log.Println("--------------------------------------------------------------------------------") + dumpPlan( + []string{"================================================================================", "plan post-optimzation"}, + plan, + "--------------------------------------------------------------------------------", + ) return result, nil } @@ -168,38 +184,36 @@ func (ta RelationAliasesMap) addAlias(alias types.IdentifiableByName, target typ return nil } +// build a map of alias names to relations func getRelationAliases(n types.PlanOperator, scope *OptimizerScope) (RelationAliasesMap, error) { - var aliases RelationAliasesMap - var aliasFn func(node types.PlanOperator) bool var inspectErr error - aliasFn = func(node types.PlanOperator) bool { + + aliases := make(RelationAliasesMap) + InspectPlan(n, func(node types.PlanOperator) bool { if node == nil { return false } - if at, ok := node.(*PlanOpRelAlias); ok { - switch t := at.ChildOp.(type) { + switch node := node.(type) { + case *PlanOpRelAlias: + switch t := node.ChildOp.(type) { case *PlanOpPQLTableScan: - inspectErr = aliases.addAlias(at, t) + inspectErr = aliases.addAlias(node, t) case *PlanOpSubquery: - inspectErr = aliases.addAlias(at, t) + inspectErr = aliases.addAlias(node, t) default: - panic(fmt.Sprintf("unexpected child node '%T'", at.ChildOp)) + inspectErr = sql3.NewErrInternalf("unexpected alias child type '%T", node.ChildOp) } return false - } - switch node := node.(type) { case *PlanOpPQLTableScan: inspectErr = aliases.addAlias(node, node) return false + } - return true - } + }) - aliases = make(RelationAliasesMap) - InspectPlan(n, aliasFn) if inspectErr != nil { return nil, inspectErr } @@ -232,6 +246,61 @@ func filterPushdownAboveTablesChildSelector(c ParentContext) bool { return true } +// when we compile and create a PlanOpPQLTableScan we just add all the columns to the underlying extract. This is a bad idea, since +// extracts are expensive, more so when we are askign for columns we don't actually need. This function removes those uneeded references. +func removeUnusedExtractColumnReferences(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { + // get all the qualifiedRefs across the plan + // using a map to eliminate dupes and we don't + // care about the order when iterating + refs := make(map[string]*qualifiedRefPlanExpression) + InspectOperatorExpressions(n, func(pe types.PlanExpression) bool { + switch qref := pe.(type) { + case *qualifiedRefPlanExpression: + refs[qref.String()] = qref + return false + } + return true + }) + + return TransformPlanOpWithParent(n, func(c ParentContext) bool { return true }, func(c ParentContext) (types.PlanOperator, bool, error) { + switch thisNode := c.Operator.(type) { + case *PlanOpPQLTableScan: + + newExtractList := make([]string, 0) + + // loop thru the extract list and make a new extract list + // with just the columns we need + alias, ok := c.Parent.(*PlanOpRelAlias) + if ok { + // handle the case where the parent is an alias + for _, ex := range thisNode.columns { + for _, ref := range refs { + if (strings.EqualFold(ref.tableName, thisNode.tableName) || strings.EqualFold(ref.tableName, alias.alias)) && strings.EqualFold(ex, ref.columnName) { + newExtractList = append(newExtractList, ex) + break + } + } + } + } else { + for _, ex := range thisNode.columns { + for _, ref := range refs { + if strings.EqualFold(ref.tableName, thisNode.tableName) && strings.EqualFold(ex, ref.columnName) { + newExtractList = append(newExtractList, ex) + break + } + } + } + } + + // newExtractList should now contain just the cols that are referenced + return NewPlanOpPQLTableScan(a, thisNode.tableName, newExtractList), false, nil + + default: + return thisNode, true, nil + } + }) +} + // returns an expression given a list of expressions, if the list is > 2 expressions, all the individual // expressions are ANDed together func joinExprsWithAnd(exprs ...types.PlanExpression) types.PlanExpression { @@ -263,43 +332,43 @@ func removePushedDownConditions(ctx context.Context, a *ExecutionPlanner, node * return NewPlanOpFilter(a, joinedExpr, node.ChildOp), false, nil } -func getRelation(node types.PlanOperator) types.IdentifiableByName { - var relation types.IdentifiableByName - InspectPlan(node, func(node types.PlanOperator) bool { - switch n := node.(type) { - case *PlanOpPQLTableScan: - relation = n - return false - } - return true - }) - return relation -} - func pushdownFiltersToFilterableRelations(ctx context.Context, a *ExecutionPlanner, tableNode types.PlanOperator, scope *OptimizerScope, filters *filterSet, tableAliases RelationAliasesMap) (types.PlanOperator, bool, error) { - // only do this if it is an alias or a pql table scan - switch tableNode.(type) { - case *PlanOpRelAlias, *PlanOpPQLTableScan: - // continue - default: - return nil, true, sql3.NewErrInternalf("unexpected op type '%T'", tableNode) - } + var table types.IdentifiableByName - table := getRelation(tableNode) - if table == nil { + // only do this if it is a pql table scan + switch rel := tableNode.(type) { + case *PlanOpPQLTableScan: + table = rel + default: return tableNode, true, nil } + // is the thing filterable? ft, ok := table.(types.FilteredRelation) if !ok { return tableNode, true, nil } // do we have any filters for this table? if not, bail... - tableFilters := filters.availableFiltersForTable(table.Name()) + availableFilters := filters.availableFiltersForTable(table.Name()) + if len(availableFilters) == 0 { + return tableNode, true, nil + } + + tableFilters := make([]types.PlanExpression, 0) + // can the filters be pushed down? + for _, tf := range availableFilters { + // try and generate a pql call graph, if we can't we can't push the filter down + _, err := a.generatePQLCallFromExpr(ctx, tf) + if err == nil { + tableFilters = append(tableFilters, tf) + } + } + // did we end up with any filters? if len(tableFilters) == 0 { return tableNode, true, nil } + filters.markFiltersHandled(tableFilters...) // fix the field refs @@ -316,8 +385,13 @@ func pushdownFiltersToFilterableRelations(ctx context.Context, a *ExecutionPlann } func pushdownFiltersToAboveRelation(ctx context.Context, a *ExecutionPlanner, tableNode types.PlanOperator, scope *OptimizerScope, filters *filterSet) (types.PlanOperator, bool, error) { - table := getRelation(tableNode) - if table == nil { + var table types.IdentifiableByName + + // only do this if it is a pql table scan + switch rel := tableNode.(type) { + case *PlanOpPQLTableScan: + table = rel + default: return tableNode, true, nil } @@ -341,7 +415,7 @@ func pushdownFiltersToAboveRelation(ctx context.Context, a *ExecutionPlanner, ta if pushedDownFilterExpression != nil { return NewPlanOpFilter(a, pushedDownFilterExpression, node), false, nil } - return node, false, nil + return node, true, nil default: return nil, true, sql3.NewErrInternalf("unexpected op type '%T'", tableNode) } @@ -354,9 +428,12 @@ func pushdownFilters(ctx context.Context, a *ExecutionPlanner, n types.PlanOpera return nil, true, err } + // push filter terms down into anything that supports being filtered directly pushdownFiltersForFilterableRelations := func(n *PlanOpFilter, filters *filterSet) (types.PlanOperator, bool, error) { return TransformPlanOpWithParent(n, filterPushdownChildSelector, func(c ParentContext) (types.PlanOperator, bool, error) { switch node := c.Operator.(type) { + + // for the filter in question remove any terms that have been pushed down case *PlanOpFilter: n, samePred, err := removePushedDownConditions(ctx, a, node, filters) if err != nil { @@ -364,6 +441,7 @@ func pushdownFilters(ctx context.Context, a *ExecutionPlanner, n types.PlanOpera } return n, samePred, nil + // PlanOpPQLTableScan supports being filtered, PlanOpRelAlias is included here as a "transparent" op case *PlanOpRelAlias, *PlanOpPQLTableScan: n, samePred, err := pushdownFiltersToFilterableRelations(ctx, a, node, scope, filters, tableAliases) if err != nil { @@ -379,6 +457,7 @@ func pushdownFilters(ctx context.Context, a *ExecutionPlanner, n types.PlanOpera pushdownFiltersCloseToRelations := func(n types.PlanOperator, filters *filterSet) (types.PlanOperator, bool, error) { return TransformPlanOpWithParent(n, filterPushdownAboveTablesChildSelector, func(c ParentContext) (types.PlanOperator, bool, error) { switch node := c.Operator.(type) { + case *PlanOpFilter: n, same, err := removePushedDownConditions(ctx, a, node, filters) if err != nil { @@ -388,6 +467,7 @@ func pushdownFilters(ctx context.Context, a *ExecutionPlanner, n types.PlanOpera return n, true, nil } return n, false, nil + case *PlanOpRelAlias, *PlanOpPQLTableScan: _, same, err := pushdownFiltersToAboveRelation(ctx, a, node, scope, filters) if err != nil { @@ -403,27 +483,34 @@ func pushdownFilters(ctx context.Context, a *ExecutionPlanner, n types.PlanOpera }) } + // look for filter ops and push the conditions within them down to things that can be filtered return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) { - switch n := node.(type) { + switch thisNode := node.(type) { case *PlanOpFilter: + + // get the filter conditions from this filter in a map by table filtersByTable := getFiltersByRelation(n) - filters := newFilterSet(n.Predicate, filtersByTable, tableAliases) + + // make a struct to hold the expression for this filter, the broken up filter conditions + // and a map of alias name to relations + filters := newFilterSet(thisNode.Predicate, filtersByTable, tableAliases) // first push down filters to any op that supports a filter - node, sameA, err := pushdownFiltersForFilterableRelations(n, filters) + newNode, sameA, err := pushdownFiltersForFilterableRelations(thisNode, filters) if err != nil { return nil, true, err } // second push down filters as close as possible to the relations they apply to - node, sameB, err := pushdownFiltersCloseToRelations(node, filters) + var sameB bool + newNode, sameB, err = pushdownFiltersCloseToRelations(newNode, filters) if err != nil { return nil, true, err } - return node, sameA && sameB, nil + return newNode, sameA && sameB, nil default: - return n, true, nil + return node, true, nil } }) } @@ -433,9 +520,9 @@ func getFiltersByRelation(n types.PlanOperator) map[string][]types.PlanExpressio filters := make(map[string][]types.PlanExpression) InspectPlan(n, func(node types.PlanOperator) bool { - switch nd := node.(type) { + switch thisNode := node.(type) { case *PlanOpFilter: - fs := exprToRelationFilters(nd.Predicate) + fs := exprToRelationFilters(thisNode.Predicate) for k, exprs := range fs { filters[k] = append(filters[k], exprs...) @@ -458,13 +545,13 @@ func exprToRelationFilters(expr types.PlanExpression) map[string][]types.PlanExp hasSubquery := false InspectExpression(expr, func(e types.PlanExpression) bool { - f, ok := e.(*qualifiedRefPlanExpression) - if ok { - if !seenTables[f.tableName] { - seenTables[f.tableName] = true - lastTable = f.tableName + switch thisExpr := e.(type) { + case *qualifiedRefPlanExpression: + if !seenTables[thisExpr.tableName] { + seenTables[thisExpr.tableName] = true + lastTable = thisExpr.tableName } - } else if _, isSubquery := e.(*subqueryPlanExpression); isSubquery { + case *subqueryPlanExpression: hasSubquery = true return false } @@ -476,7 +563,6 @@ func exprToRelationFilters(expr types.PlanExpression) map[string][]types.PlanExp filters[lastTable] = append(filters[lastTable], expr) } } - return filters } @@ -544,6 +630,28 @@ func tryToReplaceGroupByWithPQLAggregate(ctx context.Context, a *ExecutionPlanne return n, true, nil } +func tryToReplaceConstRowDeleteWithFilteredDelete(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { + return TransformPlanOp(n, func(node types.PlanOperator) (types.PlanOperator, bool, error) { + switch node := node.(type) { + case *PlanOpPQLConstRowDelete: + switch child := node.ChildOp.(type) { + case *PlanOpPQLTableScan: + if child.filter != nil { + _, err := a.generatePQLCallFromExpr(ctx, child.filter) + if err == nil { + return NewPlanOpPQLFilteredDelete(a, node.tableName, child.filter), false, nil + } + } + return node, true, nil + default: + return node, true, nil + } + default: + return node, true, nil + } + }) +} + func tryToReplaceGroupByWithPQLGroupBy(ctx context.Context, a *ExecutionPlanner, n types.PlanOperator, scope *OptimizerScope) (types.PlanOperator, bool, error) { //bail if there are any joins joins, err := hasJoins(ctx, a, n, scope) diff --git a/sql3/planner/planwalker.go b/sql3/planner/planwalker.go index bc71b82cb..8e7e50c29 100644 --- a/sql3/planner/planwalker.go +++ b/sql3/planner/planwalker.go @@ -78,8 +78,8 @@ func (f exprInspector) VisitExpr(e types.PlanExpression) ExprVisitor { return nil } -// WalkExpressions traverses the plan and calls ExprWalk on any expression it finds -func WalkExpressions(v ExprVisitor, op types.PlanOperator) { +// walkExpressions traverses the plan and calls ExprWalk on any expression it finds +func walkExpressions(v ExprVisitor, op types.PlanOperator) { InspectPlan(op, func(operator types.PlanOperator) bool { if n, ok := operator.(types.ContainsExpressions); ok { for _, e := range n.Expressions() { @@ -90,13 +90,13 @@ func WalkExpressions(v ExprVisitor, op types.PlanOperator) { }) } -// InspectExpressions traverses the plan and calls WalkExpressions on any +// InspectOperatorExpressions traverses the plan and calls WalkExpressions on any // expression it finds. -func InspectExpressions(op types.PlanOperator, f exprInspector) { - WalkExpressions(f, op) +func InspectOperatorExpressions(op types.PlanOperator, f exprInspector) { + walkExpressions(f, op) } -// InspectExpression traverses expressoins in depth-first order +// InspectExpression traverses expressions in depth-first order func InspectExpression(expr types.PlanExpression, f func(expr types.PlanExpression) bool) { ExprWalk(exprInspector(f), expr) } diff --git a/sql3/planner/types/planexpression.go b/sql3/planner/types/planexpression.go index 2a49677d7..ad99a15a4 100644 --- a/sql3/planner/types/planexpression.go +++ b/sql3/planner/types/planexpression.go @@ -51,7 +51,7 @@ type AggregationBuffer interface { Update(ctx context.Context, row Row) error } -// Interface to an expression that is a an aggregate +// interface to an expression that is a an aggregate type Aggregable interface { fmt.Stringer @@ -61,7 +61,7 @@ type Aggregable interface { AggAdditionalExpr() []PlanExpression } -// Interface to something that can be identified by a name +// interface to something that can be identified by a name type IdentifiableByName interface { Name() string } diff --git a/sql3/test/defs/defs.go b/sql3/test/defs/defs.go index 31bb62f85..162d94f3b 100644 --- a/sql3/test/defs/defs.go +++ b/sql3/test/defs/defs.go @@ -20,6 +20,8 @@ var TableTests []TableTest = []TableTest{ topTests, + deleteTests, + setLiteralTests, setFunctionTests, setParameterTests, diff --git a/sql3/test/defs/defs_delete.go b/sql3/test/defs/defs_delete.go new file mode 100644 index 000000000..68d2ff851 --- /dev/null +++ b/sql3/test/defs/defs_delete.go @@ -0,0 +1,389 @@ +// Copyright 2021 Molecula Corp. All rights reserved. +package defs + +import "time" + +func earlyMay2022() time.Time { + tm, err := time.ParseInLocation(time.RFC3339, "2022-05-05T13:00:00+00:00", time.UTC) + if err != nil { + panic(err.Error()) + } + return tm +} + +func lateMay2022() time.Time { + tm, err := time.ParseInLocation(time.RFC3339, "2022-05-28T13:00:00+00:00", time.UTC) + if err != nil { + panic(err.Error()) + } + return tm +} + +// DELETE tests +var deleteTests = TableTest{ + name: "delete_tests", + Table: tbl( + "del_all_types", + srcHdrs( + srcHdr("_id", fldTypeID), + srcHdr("i1", fldTypeInt, "min 0", "max 1000"), + srcHdr("b1", fldTypeBool), + srcHdr("d1", fldTypeDecimal2), + srcHdr("id1", fldTypeID), + srcHdr("ids1", fldTypeIDSet), + srcHdr("s1", fldTypeString), + srcHdr("ss1", fldTypeStringSet), + srcHdr("t1", fldTypeTimestamp), + ), + srcRows( + srcRow(int64(1), int64(1000), bool(true), float64(12.34), int64(20), []int64{101, 102}, string("foo"), []string{"101", "102"}, earlyMay2022()), + srcRow(int64(2), int64(1000), bool(true), float64(12.34), int64(20), []int64{101, 102}, string("foo"), []string{"101", "102"}, earlyMay2022()), + srcRow(int64(3), int64(1000), bool(true), float64(12.34), int64(20), []int64{101, 102}, string("foo"), []string{"101", "102"}, earlyMay2022()), + srcRow(int64(4), int64(1000), bool(true), float64(12.34), int64(20), []int64{101, 102}, string("foo"), []string{"101", "102"}, earlyMay2022()), + srcRow(int64(5), int64(1000), bool(true), float64(12.34), int64(20), []int64{101, 102}, string("foo"), []string{"101", "102"}, lateMay2022()), + ), + ), + SQLTests: []SQLTest{ + { + SQLs: sqls( + "delete from del_all_types where _id = 1;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + // ordering is important here - this test validates the previous delete happened + SQLs: sqls( + "select _id from del_all_types where _id = 1;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "delete from del_all_types where _id in (2, 3);", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + // ordering is important here - this test validates the previous delete happened + SQLs: sqls( + "select _id from del_all_types where _id = 2 or _id = 3;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + + // delete with in + { + SQLs: sqls( + "create table sub_query (_id id, i1 int min 0 max 1000);", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "insert into sub_query values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6);", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "delete from del_all_types where _id in (select _id from sub_query where i1 > 3) and i1 > 10;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "select _id from del_all_types where _id > 4;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + + // dates + { + SQLs: sqls( + `insert into del_all_types + values + (1,1000,true,12.34,20,[101,102],'foo',['101','102'],'2010-01-01T00:00:00Z'), + (2,1000,true,12.34,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (3,1000,true,12.34,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (4,1000,true,12.34,20,[101,102],'foo',['101','102'],'2020-01-01T00:00:00Z');`, + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "delete from del_all_types where t1 > '2010-01-01T00:00:00Z';", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "select _id from del_all_types where _id > 1;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + + // ints + { + SQLs: sqls( + `insert into del_all_types + values + (1,100,true,12.34,20,[101,102],'foo',['101','102'],'2010-01-01T00:00:00Z'), + (2,200,true,12.34,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (3,300,true,12.34,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (4,400,true,12.34,20,[101,102],'foo',['101','102'],'2020-01-01T00:00:00Z');`, + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "delete from del_all_types where i1 > 200;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "select _id from del_all_types where i1 > 200;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "delete from del_all_types where i1 < 300;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "select _id from del_all_types;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + + // bool + { + SQLs: sqls( + `insert into del_all_types + values + (1,100,true,12.34,20,[101,102],'foo',['101','102'],'2010-01-01T00:00:00Z'), + (2,200,true,12.34,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (3,300,true,12.34,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (4,400,true,12.34,20,[101,102],'foo',['101','102'],'2020-01-01T00:00:00Z');`, + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "delete from del_all_types where b1 = true;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "select _id from del_all_types;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + + // id sets + { + SQLs: sqls( + `insert into del_all_types + values + (1,100,true,12.34,20,[101,102],'foo',['101','102'],'2010-01-01T00:00:00Z'), + (2,200,true,12.34,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (3,300,true,12.34,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (4,400,true,12.34,20,[101,102,103],'foo',['101','102'],'2020-01-01T00:00:00Z');`, + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "delete from del_all_types where setcontains(ids1, 103);", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "select _id from del_all_types where _id = 4;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + + // compound expressions + { + SQLs: sqls( + `insert into del_all_types + values + (1,100,true,12.34,20,[101,102],'foo',['101','102'],'2010-01-01T00:00:00Z'), + (2,200,true,12.35,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (3,300,true,12.36,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (4,400,true,12.37,20,[101,102,103],'foo',['101','102'],'2020-01-01T00:00:00Z');`, + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "delete from del_all_types where d1 = 12.36 and i1 = 300;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "select _id from del_all_types where _id = 3;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "delete from del_all_types where d1 = 12.34 or i1 = 200;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "select _id from del_all_types where _id = 1 or _id = 2;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + + // scalar function filters + { + SQLs: sqls( + `insert into del_all_types + values + (1,100,true,12.34,20,[101,102],'foo',['101','102'],'2010-01-01T00:00:00Z'), + (2,200,true,12.35,20,[101,102],'bar',['101','102'],'2012-11-01T22:08:41Z'), + (3,300,true,12.36,20,[101,102],'zoo',['101','102'],'2012-11-01T22:08:41Z'), + (4,400,true,12.37,20,[101,102,103],'raz',['101','102','103'],'2020-01-01T00:00:00Z');`, + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "delete from del_all_types where substring(s1, 0, 1) = 'f';", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "select _id from del_all_types where _id = 1;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + + // delete everything + { + SQLs: sqls( + `insert into del_all_types + values + (1,100,true,12.34,20,[101,102],'foo',['101','102'],'2010-01-01T00:00:00Z'), + (2,200,true,12.34,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (3,300,true,12.34,20,[101,102],'foo',['101','102'],'2012-11-01T22:08:41Z'), + (4,400,true,12.34,20,[101,102],'foo',['101','102'],'2020-01-01T00:00:00Z');`, + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "delete from del_all_types;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "select _id from del_all_types;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, + }, + }, +} diff --git a/sql3/test/defs/defs_null.go b/sql3/test/defs/defs_null.go index 5b65478d1..1d8ec6c5d 100644 --- a/sql3/test/defs/defs_null.go +++ b/sql3/test/defs/defs_null.go @@ -302,13 +302,24 @@ var nullFilterTests = TableTest{ SQLs: sqls( "select _id from null_filter_all_types where _id is null", ), - ExpErr: "'_id' column cannot be used in a is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows(), + Compare: CompareExactUnordered, }, { SQLs: sqls( "select _id from null_filter_all_types where _id is not null", ), - ExpErr: "'_id' column cannot be used in a is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows( + row(int64(1)), + row(int64(2)), + ), + Compare: CompareExactUnordered, }, { SQLs: sqls( @@ -338,13 +349,25 @@ var nullFilterTests = TableTest{ SQLs: sqls( "select _id from null_filter_all_types where b1 is null", ), - ExpErr: "unsupported type 'bool' for is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows( + row(int64(1)), + ), + Compare: CompareExactUnordered, }, { SQLs: sqls( "select _id from null_filter_all_types where b1 is not null", ), - ExpErr: "unsupported type 'bool' for is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows( + row(int64(2)), + ), + Compare: CompareExactUnordered, }, { SQLs: sqls( @@ -374,49 +397,97 @@ var nullFilterTests = TableTest{ SQLs: sqls( "select _id from null_filter_all_types where id1 is null", ), - ExpErr: "unsupported type 'id' for is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows( + row(int64(1)), + ), + Compare: CompareExactUnordered, }, { SQLs: sqls( "select _id from null_filter_all_types where id1 is not null", ), - ExpErr: "unsupported type 'id' for is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows( + row(int64(2)), + ), + Compare: CompareExactUnordered, }, { SQLs: sqls( "select _id from null_filter_all_types where ids1 is null", ), - ExpErr: "unsupported type 'idset' for is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows( + row(int64(1)), + ), + Compare: CompareExactUnordered, }, { SQLs: sqls( "select _id from null_filter_all_types where ids1 is not null", ), - ExpErr: "unsupported type 'idset' for is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows( + row(int64(2)), + ), + Compare: CompareExactUnordered, }, { SQLs: sqls( "select _id from null_filter_all_types where s1 is null", ), - ExpErr: "unsupported type 'string' for is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows( + row(int64(1)), + ), + Compare: CompareExactUnordered, }, { SQLs: sqls( "select _id from null_filter_all_types where s1 is not null", ), - ExpErr: "unsupported type 'string' for is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows( + row(int64(2)), + ), + Compare: CompareExactUnordered, }, { SQLs: sqls( "select _id from null_filter_all_types where ss1 is null", ), - ExpErr: "unsupported type 'stringset' for is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows( + row(int64(1)), + ), + Compare: CompareExactUnordered, }, { SQLs: sqls( "select _id from null_filter_all_types where ss1 is not null", ), - ExpErr: "unsupported type 'stringset' for is/is not null filter expression", + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + ), + ExpRows: rows( + row(int64(2)), + ), + Compare: CompareExactUnordered, }, { SQLs: sqls(