diff --git a/dax/test/dax/dax_test.go b/dax/test/dax/dax_test.go index 1e9d0d215..637345bc8 100644 --- a/dax/test/dax/dax_test.go +++ b/dax/test/dax/dax_test.go @@ -144,7 +144,8 @@ func TestDAXIntegration(t *testing.T) { "testinsert/test-5", // error messages differ "percentile_test/test-6", // related to TODO in orchestrator.executePercentile "alterTable/alterTableBadTable", // looks like table does not exist is a different error in DAX - "top-tests/test-1", // don't know why this is failing at all + "top-limit-tests/test-2", // don't know why this is failing at all + "top-limit-tests/test-3", // don't know why this is failing at all "delete_tests", "viewtests/drop-view", // drop view does a delete "viewtests/drop-view-if-exists-after-drop", diff --git a/go.mod b/go.mod index c218b2e7d..0006daf48 100644 --- a/go.mod +++ b/go.mod @@ -137,6 +137,7 @@ require ( github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d // indirect github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e // indirect github.com/tinylib/msgp v1.1.2 // indirect + gonum.org/v1/gonum v0.11.0 // indirect ) require ( @@ -197,6 +198,7 @@ require ( github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/sajari/regression v1.0.1 github.com/sirupsen/logrus v1.9.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/afero v1.6.0 // indirect diff --git a/go.sum b/go.sum index e90e72013..e442c2b33 100644 --- a/go.sum +++ b/go.sum @@ -1056,6 +1056,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sajari/regression v1.0.1 h1:iTVc6ZACGCkoXC+8NdqH5tIreslDTT/bXxT6OmHR5PE= +github.com/sajari/regression v1.0.1/go.mod h1:NeG/XTW1lYfGY7YV/Z0nYDV/RGh3wxwd1yW46835flM= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -1684,6 +1686,7 @@ golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNq gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= diff --git a/sql3/errors.go b/sql3/errors.go index 6ed3ebcaa..3bc42cf6a 100644 --- a/sql3/errors.go +++ b/sql3/errors.go @@ -13,10 +13,13 @@ const ( ErrUnsupported errors.Code = "ErrUnsupported" ErrCacheKeyNotFound errors.Code = "ErrCacheKeyNotFound" - ErrDuplicateColumn errors.Code = "ErrDuplicateColumn" - ErrUnknownType errors.Code = "ErrUnknownType" - ErrUnknownIdentifier errors.Code = "ErrUnknownIdentifier" + // syntax/semantic errors + ErrDuplicateColumn errors.Code = "ErrDuplicateColumn" + ErrUnknownType errors.Code = "ErrUnknownType" + ErrUnknownIdentifier errors.Code = "ErrUnknownIdentifier" + ErrTopLimitCannotCoexist errors.Code = "ErrTopLimitCannotCoexist" + // type related errors ErrTypeIncompatibleWithBitwiseOperator errors.Code = "ErrTypeIncompatibleWithBitwiseOperator" ErrTypeIncompatibleWithLogicalOperator errors.Code = "ErrTypeIncompatibleWithLogicalOperator" ErrTypeIncompatibleWithEqualityOperator errors.Code = "ErrTypeIncompatibleWithEqualityOperator" @@ -40,8 +43,6 @@ const ( ErrTimeQuantumExpressionExpected errors.Code = "ErrTimeQuantumExpressionExpected" ErrSingleRowExpected errors.Code = "ErrSingleRowExpected" - // type related errors - // decimal ErrDecimalScaleExpected errors.Code = "ErrDecimalScaleExpected" @@ -94,6 +95,9 @@ const ( ErrViewExists errors.Code = "ErrViewExists" ErrViewNotFound errors.Code = "ErrViewNotFound" + ErrModelExists errors.Code = "ErrModelExists" + ErrModelNotFound errors.Code = "ErrModelNotFound" + ErrBadColumnConstraint errors.Code = "ErrBadColumnConstraint" ErrConflictingColumnConstraint errors.Code = "ErrConflictingColumnConstraint" @@ -141,6 +145,9 @@ const ( ErrInvalidDatetimePart errors.Code = "ErrInvalidDatetimePart" ErrOutputValueOutOfRange errors.Code = "ErrOutputValueOutOfRange" ErrDivideByZero errors.Code = "ErrDivideByZero" + + // remote execution + ErrRemoteUnauthorized errors.Code = "ErrRemoteUnauthorized" ) func NewErrDuplicateColumn(line int, col int, column string) error { @@ -164,6 +171,13 @@ func NewErrUnknownIdentifier(line int, col int, ident string) error { ) } +func NewErrErrTopLimitCannotCoexist(line int, col int) error { + return errors.New( + ErrTopLimitCannotCoexist, + fmt.Sprintf("[%d:%d] TOP and LIMIT cannot cannot be used at the same time (TOP will be deprecated in a future release)", line, col), + ) +} + func NewErrInternal(msg string) error { preamble := "internal error" _, filename, line, ok := runtime.Caller(1) @@ -647,6 +661,20 @@ func NewErrViewExists(line, col int, viewName string) error { ) } +func NewErrModelNotFound(line, col int, viewName string) error { + return errors.New( + ErrModelNotFound, + fmt.Sprintf("[%d:%d] model '%s' not found", line, col, viewName), + ) +} + +func NewErrModelExists(line, col int, viewName string) error { + return errors.New( + ErrModelExists, + fmt.Sprintf("[%d:%d] model '%s' already exists", line, col, viewName), + ) +} + func NewErrBadColumnConstraint(line, col int, constraint, columnType string) error { return errors.New( ErrBadColumnConstraint, @@ -876,3 +904,10 @@ func NewErrDivideByZero(line, col int) error { fmt.Sprintf("[%d:%d] divisor is equal to zero", line, col), ) } + +func NewErrRemoteUnauthorized(line, col int, remoteUrl string) error { + return errors.New( + ErrRemoteUnauthorized, + fmt.Sprintf("unauthorized on remote server '%s'", remoteUrl), + ) +} diff --git a/sql3/parser/ast.go b/sql3/parser/ast.go index dd5501105..3c34d0ca3 100644 --- a/sql3/parser/ast.go +++ b/sql3/parser/ast.go @@ -34,11 +34,13 @@ func (*CaseExpr) node() {} func (*CastExpr) node() {} func (*CheckConstraint) node() {} func (*ColumnDefinition) node() {} +func (*CopyStatement) node() {} func (*CommitStatement) node() {} func (*CreateDatabaseStatement) node() {} func (*CreateIndexStatement) node() {} func (*CreateTableStatement) node() {} func (*CreateFunctionStatement) node() {} +func (*CreateModelStatement) node() {} func (*CreateViewStatement) node() {} func (*DateLit) node() {} func (*DefaultConstraint) node() {} @@ -48,6 +50,7 @@ func (*DropIndexStatement) node() {} func (*DropTableStatement) node() {} func (*DropFunctionStatement) node() {} func (*DropViewStatement) node() {} +func (*DropModelStatement) node() {} func (*Exists) node() {} func (*ExplainStatement) node() {} func (*ExprList) node() {} @@ -73,12 +76,14 @@ func (*OnConstraint) node() {} func (*OrderingTerm) node() {} func (*OverClause) node() {} func (*ParenExpr) node() {} +func (*PredictStatement) node() {} func (*SetLiteralExpr) node() {} func (*ParenSource) node() {} func (*PrimaryKeyConstraint) node() {} func (*QualifiedRef) node() {} func (*QualifiedTableName) node() {} func (*Range) node() {} +func (*ReturnStatement) node() {} func (*ReleaseStatement) node() {} func (*ResultColumn) node() {} func (*RollbackStatement) node() {} @@ -111,6 +116,7 @@ func (*AlterTableStatement) stmt() {} func (*AlterViewStatement) stmt() {} func (*AnalyzeStatement) stmt() {} func (*BeginStatement) stmt() {} +func (*CopyStatement) stmt() {} func (*BulkInsertStatement) stmt() {} func (*ShowDatabasesStatement) stmt() {} func (*ShowTablesStatement) stmt() {} @@ -121,6 +127,7 @@ func (*CreateDatabaseStatement) stmt() {} func (*CreateIndexStatement) stmt() {} func (*CreateTableStatement) stmt() {} func (*CreateFunctionStatement) stmt() {} +func (*CreateModelStatement) stmt() {} func (*CreateViewStatement) stmt() {} func (*DeleteStatement) stmt() {} func (*DropDatabaseStatement) stmt() {} @@ -128,9 +135,12 @@ func (*DropIndexStatement) stmt() {} func (*DropTableStatement) stmt() {} func (*DropFunctionStatement) stmt() {} func (*DropViewStatement) stmt() {} +func (*DropModelStatement) stmt() {} +func (*PredictStatement) stmt() {} func (*ExplainStatement) stmt() {} func (*InsertStatement) stmt() {} func (*ReleaseStatement) stmt() {} +func (*ReturnStatement) stmt() {} func (*RollbackStatement) stmt() {} func (*SavepointStatement) stmt() {} func (*SelectStatement) stmt() {} @@ -177,6 +187,8 @@ func CloneStatement(stmt Statement) Statement { return stmt.Clone() case *DropViewStatement: return stmt.Clone() + case *DropModelStatement: + return stmt.Clone() case *ExplainStatement: return stmt.Clone() case *InsertStatement: @@ -2910,6 +2922,35 @@ func (s *DropViewStatement) String() string { return buf.String() } +type DropModelStatement struct { + Drop Pos // position of DROP keyword + Model Pos // position of MODEL keyword + If Pos // position of IF keyword + IfExists Pos // position of EXISTS keyword after IF + Name *Ident // view name +} + +// Clone returns a deep copy of s. +func (s *DropModelStatement) Clone() *DropModelStatement { + if s == nil { + return nil + } + other := *s + other.Name = s.Name.Clone() + return &other +} + +// String returns the string representation of the statement. +func (s *DropModelStatement) String() string { + var buf bytes.Buffer + buf.WriteString("DROP MODEL") + if s.IfExists.IsValid() { + buf.WriteString(" IF EXISTS") + } + fmt.Fprintf(&buf, " %s", s.Name.String()) + return buf.String() +} + type CreateIndexStatement struct { Create Pos // position of CREATE keyword Unique Pos // position of optional UNIQUE keyword @@ -2998,6 +3039,11 @@ func (s *DropIndexStatement) String() string { return buf.String() } +type FunctionOptionDefinition struct { + Name *Ident // option name + OptionExpr Expr // option expression +} + type ParameterDefinition struct { Name *Variable // parameter name Type *Type // data type @@ -3015,8 +3061,11 @@ type CreateFunctionStatement struct { Parameters []*ParameterDefinition // parameters Rparen Pos // position of parameter RParen - Returns Pos // position of RETURNS keyword - ReturnDef *ParameterDefinition // return def + Returns Pos // position of RETURNS keyword + ReturnType *Type // return def + + With Pos // position of WITH keyword + Options []*FunctionOptionDefinition // options As Pos // position of AS keyword @@ -3057,7 +3106,21 @@ func (s *CreateFunctionStatement) String() string { } buf.WriteString(" RETURNS ") - fmt.Fprintf(&buf, "%s %s", s.ReturnDef.Name, s.ReturnDef.Type.Name) + fmt.Fprintf(&buf, "%s", s.ReturnType.String()) + + if s.With.IsValid() { + buf.WriteString(" WITH ") + if len(s.Options) > 0 { + buf.WriteString(" (") + for idx, p := range s.Options { + if idx > 0 { + buf.WriteString(", ") + } + fmt.Fprintf(&buf, "%s %s", p.Name.Name, p.OptionExpr.String()) + } + buf.WriteString(")") + } + } buf.WriteString(" AS BEGIN") for i := range s.Body { @@ -3096,6 +3159,86 @@ func (s *DropFunctionStatement) String() string { return buf.String() } +type ReturnStatement struct { + Return Pos // position of RETURN keyword + ReturnExpr Expr // what we are returning +} + +// Clone returns a deep copy of s. +func (s *ReturnStatement) Clone() *ReturnStatement { + if s == nil { + return nil + } + other := *s + other.ReturnExpr = CloneExpr(s.ReturnExpr) + return &other +} + +func (s *ReturnStatement) String() string { + var buf bytes.Buffer + buf.WriteString("RETURN") + fmt.Fprintf(&buf, " %s", s.ReturnExpr.String()) + return buf.String() +} + +type ModelOptionDefinition struct { + Name *Ident // option name + OptionExpr Expr // option expression +} + +type CreateModelStatement struct { + Create Pos // position of CREATE keyword + Model Pos // position of MODEL keyword + If Pos // position of IF keyword + IfNot Pos // position of NOT keyword after IF + IfNotExists Pos // position of EXISTS keyword after IF NOT + Name *Ident // model name + With Pos // position of WITH keyword + + Options []*ModelOptionDefinition // options + + As Pos // position of AS keyword + + ModelQuery *SelectStatement // model query +} + +// Clone returns a deep copy of s. +func (s *CreateModelStatement) Clone() *CreateModelStatement { + if s == nil { + return nil + } + other := *s + other.Name = s.Name.Clone() + other.ModelQuery = s.ModelQuery.Clone() + return &other +} + +// String returns the string representation of the statement. +func (s *CreateModelStatement) String() string { + var buf bytes.Buffer + buf.WriteString("CREATE MODEL") + if s.IfNotExists.IsValid() { + buf.WriteString(" IF NOT EXISTS") + } + fmt.Fprintf(&buf, " %s", s.Name.String()) + + if len(s.Options) > 0 { + buf.WriteString(" (") + for idx, p := range s.Options { + if idx > 0 { + buf.WriteString(", ") + } + fmt.Fprintf(&buf, "%s %s", p.Name.Name, p.OptionExpr.String()) + } + buf.WriteString(")") + } + + buf.WriteString(" AS ") + buf.WriteString(s.ModelQuery.String()) + + return buf.String() +} + type BulkInsertMapDefinition struct { Name *Ident // map name Type *Type // data type @@ -3645,6 +3788,77 @@ func (c *IndexedColumn) String() string { return c.X.String() } +type CopyStatement struct { + Copy Pos // position of COPY keyword + Source Source // source table + To Pos // position of TO keyword + TargetName *Ident // target table name + Where Pos // position of WHERE keyword + WhereExpr Expr // where clause expression + With Pos // position of WITH keyword + + Url Expr // url for target server + ApiKey Expr // apikey for target server +} + +func (c *CopyStatement) Clone() *CopyStatement { + if c == nil { + return nil + } + other := *c + other.Source = CloneSource(c.Source) + other.TargetName = c.TargetName.Clone() + other.WhereExpr = CloneExpr(c.WhereExpr) + other.Url = CloneExpr(c.Url) + other.ApiKey = CloneExpr(c.ApiKey) + return &other +} + +func (c *CopyStatement) String() string { + var buf bytes.Buffer + + fmt.Fprintf(&buf, "COPY %s to %s", c.Source.String(), c.TargetName.String()) + if c.WhereExpr != nil { + fmt.Fprintf(&buf, " WHERE %s", c.WhereExpr.String()) + } + if c.With.IsValid() { + fmt.Fprintf(&buf, " WITH") + if c.Url != nil { + fmt.Fprintf(&buf, " URL %s", c.Url.String()) + } + if c.ApiKey != nil { + fmt.Fprintf(&buf, " APIKEY %s", c.Url.String()) + } + } + return buf.String() +} + +type PredictStatement struct { + Predict Pos // position of PREDICT keyword + Using Pos // position of USING keyword + ModelName *Ident // model name + + InputQuery *SelectStatement // input query +} + +func (c *PredictStatement) Clone() *PredictStatement { + if c == nil { + return nil + } + other := *c + other.ModelName = c.ModelName.Clone() + other.InputQuery = c.InputQuery.Clone() + return &other +} + +func (c *PredictStatement) String() string { + var buf bytes.Buffer + + fmt.Fprintf(&buf, "PREDICT USING %s", c.ModelName.String()) + fmt.Fprintf(&buf, " %s", c.InputQuery.String()) + return buf.String() +} + type SelectStatement struct { WithClause *WithClause // clause containing CTEs @@ -3685,6 +3899,8 @@ type SelectStatement struct { OrderBy Pos // position of BY keyword after ORDER OrderingTerms []*OrderingTerm // terms of ORDER BY clause + Limit Pos // position of LIMIT keyword + LimitExpr Expr // LIMIT expr } // Clone returns a deep copy of s. @@ -3694,7 +3910,6 @@ func (s *SelectStatement) Clone() *SelectStatement { } other := *s other.WithClause = s.WithClause.Clone() - //other.ValueLists = cloneExprLists(s.ValueLists) other.TopExpr = CloneExpr(s.TopExpr) other.Columns = cloneResultColumns(s.Columns) other.Source = CloneSource(s.Source) @@ -3704,6 +3919,7 @@ func (s *SelectStatement) Clone() *SelectStatement { other.Windows = cloneWindows(s.Windows) other.Compound = s.Compound.Clone() other.OrderingTerms = cloneOrderingTerms(s.OrderingTerms) + other.LimitExpr = CloneExpr(s.LimitExpr) return &other } @@ -3742,29 +3958,10 @@ func (s *SelectStatement) String() string { buf.WriteString(" ") } - /*if len(s.ValueLists) > 0 { - buf.WriteString("VALUES ") - for i, exprs := range s.ValueLists { - if i != 0 { - buf.WriteString(", ") - } - - buf.WriteString("(") - for j, expr := range exprs.Exprs { - if j != 0 { - buf.WriteString(", ") - } - buf.WriteString(expr.String()) - } - buf.WriteString(")") - } - } else {*/ buf.WriteString("SELECT ") if s.Distinct.IsValid() { buf.WriteString("DISTINCT ") - } //else if s.All.IsValid() { - // buf.WriteString("ALL ") - //} + } if s.Top.IsValid() { fmt.Fprintf(&buf, "TOP(%s) ", s.TopExpr.String()) } @@ -3810,7 +4007,6 @@ func (s *SelectStatement) String() string { buf.WriteString(window.String()) } } - // } // Write compound operator. if s.Compound != nil { @@ -3840,6 +4036,10 @@ func (s *SelectStatement) String() string { } } + if s.Limit.IsValid() { + fmt.Fprintf(&buf, "LIMIT %s", s.LimitExpr.String()) + } + return buf.String() } diff --git a/sql3/parser/ast_test.go b/sql3/parser/ast_test.go index 0c921b173..44ce35e6b 100644 --- a/sql3/parser/ast_test.go +++ b/sql3/parser/ast_test.go @@ -404,11 +404,8 @@ func TestCreateFunctionStatement_String(t *testing.T) { Type: &parser.Type{Name: &parser.Ident{Name: "int"}}, }, }, - ReturnDef: &parser.ParameterDefinition{ - Name: &parser.Variable{Name: "@scalar"}, - Type: &parser.Type{Name: &parser.Ident{Name: "int"}}, - }, - }, `CREATE FUNCTION func (@param1 int) RETURNS @scalar int AS BEGIN END`) + ReturnType: &parser.Type{Name: &parser.Ident{Name: "int"}}, + }, `CREATE FUNCTION func (@param1 int) RETURNS int AS BEGIN END`) AssertStatementStringer(t, &parser.CreateFunctionStatement{ IfNotExists: pos(0), @@ -419,11 +416,8 @@ func TestCreateFunctionStatement_String(t *testing.T) { Type: &parser.Type{Name: &parser.Ident{Name: "int"}}, }, }, - ReturnDef: &parser.ParameterDefinition{ - Name: &parser.Variable{Name: "@scalar"}, - Type: &parser.Type{Name: &parser.Ident{Name: "int"}}, - }, - }, `CREATE FUNCTION IF NOT EXISTS func (@param1 int) RETURNS @scalar int AS BEGIN END`) + ReturnType: &parser.Type{Name: &parser.Ident{Name: "int"}}, + }, `CREATE FUNCTION IF NOT EXISTS func (@param1 int) RETURNS int AS BEGIN END`) } func TestCreateViewStatement_String(t *testing.T) { @@ -866,34 +860,34 @@ func TestSelectStatement_String(t *testing.T) { }, }, `SELECT * FROM (SELECT *)`) - AssertStatementStringer(t, &parser.SelectStatement{ - Columns: []*parser.ResultColumn{{Star: pos(0)}}, - Source: &parser.QualifiedTableName{Name: &parser.Ident{Name: "tbl"}}, - Windows: []*parser.Window{ - { - Name: &parser.Ident{Name: "win1"}, - Definition: &parser.WindowDefinition{ - Base: &parser.Ident{Name: "base"}, - Partitions: []parser.Expr{&parser.Ident{Name: "x"}, &parser.Ident{Name: "y"}}, - OrderingTerms: []*parser.OrderingTerm{ - {X: &parser.Ident{Name: "x"}, Asc: pos(0), NullsFirst: pos(0)}, - {X: &parser.Ident{Name: "y"}, Desc: pos(0), NullsLast: pos(0)}, - }, - Frame: &parser.FrameSpec{ - Range: pos(0), - UnboundedX: pos(0), - PrecedingX: pos(0), - }, - }, - }, - { - Name: &parser.Ident{Name: "win2"}, - Definition: &parser.WindowDefinition{ - Base: &parser.Ident{Name: "base2"}, - }, - }, - }, - }, `SELECT * FROM tbl WINDOW win1 AS (base PARTITION BY x, y ORDER BY x ASC NULLS FIRST, y DESC NULLS LAST RANGE UNBOUNDED PRECEDING), win2 AS (base2)`) + // AssertStatementStringer(t, &parser.SelectStatement{ + // Columns: []*parser.ResultColumn{{Star: pos(0)}}, + // Source: &parser.QualifiedTableName{Name: &parser.Ident{Name: "tbl"}}, + // Windows: []*parser.Window{ + // { + // Name: &parser.Ident{Name: "win1"}, + // Definition: &parser.WindowDefinition{ + // Base: &parser.Ident{Name: "base"}, + // Partitions: []parser.Expr{&parser.Ident{Name: "x"}, &parser.Ident{Name: "y"}}, + // OrderingTerms: []*parser.OrderingTerm{ + // {X: &parser.Ident{Name: "x"}, Asc: pos(0), NullsFirst: pos(0)}, + // {X: &parser.Ident{Name: "y"}, Desc: pos(0), NullsLast: pos(0)}, + // }, + // Frame: &parser.FrameSpec{ + // Range: pos(0), + // UnboundedX: pos(0), + // PrecedingX: pos(0), + // }, + // }, + // }, + // { + // Name: &parser.Ident{Name: "win2"}, + // Definition: &parser.WindowDefinition{ + // Base: &parser.Ident{Name: "base2"}, + // }, + // }, + // }, + // }, `SELECT * FROM tbl WINDOW win1 AS (base PARTITION BY x, y ORDER BY x ASC NULLS FIRST, y DESC NULLS LAST RANGE UNBOUNDED PRECEDING), win2 AS (base2)`) // AssertStatementStringer(t, &sql.SelectStatement{ // WithClause: &sql.WithClause{ @@ -914,38 +908,38 @@ func TestSelectStatement_String(t *testing.T) { // }, // }, `WITH "cte" ("x", "y") AS (SELECT *) VALUES (1, 2), (3, 4)`) - AssertStatementStringer(t, &parser.SelectStatement{ - Columns: []*parser.ResultColumn{{Star: pos(0)}}, - Union: pos(0), - Compound: &parser.SelectStatement{ - Columns: []*parser.ResultColumn{{Star: pos(0)}}, - }, - }, `SELECT * UNION SELECT *`) + // AssertStatementStringer(t, &parser.SelectStatement{ + // Columns: []*parser.ResultColumn{{Star: pos(0)}}, + // Union: pos(0), + // Compound: &parser.SelectStatement{ + // Columns: []*parser.ResultColumn{{Star: pos(0)}}, + // }, + // }, `SELECT * UNION SELECT *`) - AssertStatementStringer(t, &parser.SelectStatement{ - Columns: []*parser.ResultColumn{{Star: pos(0)}}, - Union: pos(0), - UnionAll: pos(0), - Compound: &parser.SelectStatement{ - Columns: []*parser.ResultColumn{{Star: pos(0)}}, - }, - }, `SELECT * UNION ALL SELECT *`) + // AssertStatementStringer(t, &parser.SelectStatement{ + // Columns: []*parser.ResultColumn{{Star: pos(0)}}, + // Union: pos(0), + // UnionAll: pos(0), + // Compound: &parser.SelectStatement{ + // Columns: []*parser.ResultColumn{{Star: pos(0)}}, + // }, + // }, `SELECT * UNION ALL SELECT *`) - AssertStatementStringer(t, &parser.SelectStatement{ - Columns: []*parser.ResultColumn{{Star: pos(0)}}, - Intersect: pos(0), - Compound: &parser.SelectStatement{ - Columns: []*parser.ResultColumn{{Star: pos(0)}}, - }, - }, `SELECT * INTERSECT SELECT *`) + // AssertStatementStringer(t, &parser.SelectStatement{ + // Columns: []*parser.ResultColumn{{Star: pos(0)}}, + // Intersect: pos(0), + // Compound: &parser.SelectStatement{ + // Columns: []*parser.ResultColumn{{Star: pos(0)}}, + // }, + // }, `SELECT * INTERSECT SELECT *`) - AssertStatementStringer(t, &parser.SelectStatement{ - Columns: []*parser.ResultColumn{{Star: pos(0)}}, - Except: pos(0), - Compound: &parser.SelectStatement{ - Columns: []*parser.ResultColumn{{Star: pos(0)}}, - }, - }, `SELECT * EXCEPT SELECT *`) + // AssertStatementStringer(t, &parser.SelectStatement{ + // Columns: []*parser.ResultColumn{{Star: pos(0)}}, + // Except: pos(0), + // Compound: &parser.SelectStatement{ + // Columns: []*parser.ResultColumn{{Star: pos(0)}}, + // }, + // }, `SELECT * EXCEPT SELECT *`) AssertStatementStringer(t, &parser.SelectStatement{ Columns: []*parser.ResultColumn{{Star: pos(0)}}, diff --git a/sql3/parser/parser.go b/sql3/parser/parser.go index 3a8ff4e65..48e4fbcc5 100644 --- a/sql3/parser/parser.go +++ b/sql3/parser/parser.go @@ -45,10 +45,10 @@ func (p *Parser) ParseStatement() (stmt Statement, err error) { switch tok := p.peek(); tok { case EOF: return nil, io.EOF - //case EXPLAIN: - // if stmt, err = p.parseExplainStatement(); err != nil { - // return stmt, err - // } + case EXPLAIN: + if stmt, err = p.parseExplainStatement(); err != nil { + return stmt, err + } default: if stmt, err = p.parseNonExplainStatement(); err != nil { return stmt, err @@ -64,7 +64,6 @@ func (p *Parser) ParseStatement() (stmt Statement, err error) { return stmt, nil } -/* // parseExplain parses EXPLAIN [QUERY PLAN] STMT. func (p *Parser) parseExplainStatement() (_ *ExplainStatement, err error) { var tok Token @@ -89,7 +88,7 @@ func (p *Parser) parseExplainStatement() (_ *ExplainStatement, err error) { return &stmt, err } return &stmt, nil -}*/ +} // parseStmt parses all statement types. func (p *Parser) parseNonExplainStatement() (Statement, error) { @@ -102,10 +101,14 @@ func (p *Parser) parseNonExplainStatement() (Statement, error) { return p.parseBulkInsertStatement() case CREATE: return p.parseCreateStatement() + case COPY: + return p.parseCopyStatement() case DROP: return p.parseDropStatement() case SELECT: return p.parseSelectStatement(false, nil) + case PREDICT: + return p.parsePredictStatement() case INSERT, REPLACE: return p.parseInsertStatement(nil) case UPDATE: @@ -337,8 +340,10 @@ func (p *Parser) parseCreateStatement() (Statement, error) { return p.parseCreateIndexStatement(pos)*/ case FUNCTION: return p.parseCreateFunctionStatement(pos) + case MODEL: + return p.parseCreateModelStatement(pos) default: - return nil, p.errorExpected(pos, tok, "DATABASE, TABLE, VIEW or FUNCTION") + return nil, p.errorExpected(pos, tok, "DATABASE, TABLE, VIEW, FUNCTION or MODEL") } } @@ -373,6 +378,8 @@ func (p *Parser) parseDropStatement() (Statement, error) { return p.parseDropIndexStatement(pos)*/ case FUNCTION: return p.parseDropFunctionStatement(pos) + case MODEL: + return p.parseDropModelStatement(pos) default: return nil, p.errorExpected(pos, tok, "DATABASE, TABLE, VIEW or FUNCTION") } @@ -1158,6 +1165,72 @@ func (p *Parser) parseDropTableStatement(dropPos Pos) (_ *DropTableStatement, er return &stmt, nil } +func (p *Parser) parseCopyStatement() (_ *CopyStatement, err error) { + assert(p.peek() == COPY) + + var stmt CopyStatement + stmt.Copy, _, _ = p.scan() + + ident, err := p.parseIdent("table name") + if err != nil { + return &stmt, err + } + + if stmt.Source, err = p.parseQualifiedTableName(ident); err != nil { + return &stmt, err + } + + if p.peek() != TO { + return &stmt, p.errorExpected(p.pos, p.tok, "TO") + } + stmt.To, _, _ = p.scan() + + if stmt.TargetName, err = p.parseIdent("table name"); err != nil { + return &stmt, err + } + + // parse optional "WHERE expr" + if p.peek() == WHERE { + stmt.Where, _, _ = p.scan() + if stmt.WhereExpr, err = p.ParseExpr(); err != nil { + return &stmt, err + } + } + + // options + if p.peek() == WITH { + stmt.With, _, _ = p.scan() + if !isCopyOptionStartToken(p.peek(), p) { + return &stmt, p.errorExpected(p.pos, p.tok, "URL or APIKEY") + } + + for { + option, err := p.parseIdent("copy option") + if err != nil { + return &stmt, err + } + + switch strings.ToLower(option.Name) { + case "url": + stmt.Url, err = p.ParseExpr() + if err != nil { + return &stmt, err + } + + case "apikey": + stmt.ApiKey, err = p.ParseExpr() + if err != nil { + return &stmt, err + } + } + if !isCopyOptionStartToken(p.peek(), p) { + break + } + } + } + return &stmt, nil +} + func (p *Parser) parseCreateViewStatement(createPos Pos) (_ *CreateViewStatement, err error) { assert(p.peek() == VIEW) @@ -1286,6 +1359,29 @@ func (p *Parser) parseDropViewStatement(dropPos Pos) (_ *DropViewStatement, err return &stmt, nil } +func (p *Parser) parseDropModelStatement(dropPos Pos) (_ *DropModelStatement, err error) { + assert(p.peek() == MODEL) + + var stmt DropModelStatement + stmt.Drop = dropPos + stmt.Model, _, _ = p.scan() + + // Parse optional "IF EXISTS". + if p.peek() == IF { + stmt.If, _, _ = p.scan() + if p.peek() != EXISTS { + return &stmt, p.errorExpected(p.pos, p.tok, "EXISTS") + } + stmt.IfExists, _, _ = p.scan() + } + + if stmt.Name, err = p.parseIdent("view name"); err != nil { + return &stmt, err + } + + return &stmt, nil +} + /*func (p *Parser) parseCreateIndexStatement(createPos Pos) (_ *CreateIndexStatement, err error) { assert(p.peek() == INDEX || p.peek() == UNIQUE) @@ -1457,11 +1553,35 @@ func (p *Parser) parseCreateFunctionStatement(createPos Pos) (_ *CreateFunctionS return &stmt, p.errorExpected(p.pos, p.tok, "RETURNS") } stmt.Returns, _, _ = p.scan() - stmt.ReturnDef, err = p.parseParameterDefinition() + stmt.ReturnType, err = p.parseType() if err != nil { return &stmt, err } + // options + if p.peek() == WITH { + stmt.With, _, _ = p.scan() + stmt.Options = make([]*FunctionOptionDefinition, 0) + for { + option, err := p.parseIdent("function option") + if err != nil { + return &stmt, err + } + + expr, err := p.ParseExpr() + if err != nil { + return &stmt, err + } + stmt.Options = append(stmt.Options, &FunctionOptionDefinition{ + Name: option, + OptionExpr: expr, + }) + if p.peek() == AS { + break + } + } + } + if p.peek() != AS { return &stmt, p.errorExpected(p.pos, p.tok, "AS") } @@ -1473,7 +1593,7 @@ func (p *Parser) parseCreateFunctionStatement(createPos Pos) (_ *CreateFunctionS stmt.Begin, _, _ = p.scan() for { - s, err := p.parseFunctionBodyStatement() + s, err := p.parseFunctionBodyStatement(&stmt) if err != nil { return &stmt, err } @@ -1494,8 +1614,15 @@ func (p *Parser) parseCreateFunctionStatement(createPos Pos) (_ *CreateFunctionS return &stmt, nil } -func (p *Parser) parseFunctionBodyStatement() (stmt Statement, err error) { +func (p *Parser) parseFunctionBodyStatement(cf *CreateFunctionStatement) (stmt Statement, err error) { switch p.peek() { + case RETURN: + s, err := p.parseReturnStatement() + if err != nil { + return stmt, err + } + cf.Body = append(cf.Body, s) + case END: break default: @@ -1507,6 +1634,20 @@ func (p *Parser) parseFunctionBodyStatement() (stmt Statement, err error) { return stmt, nil } +func (p *Parser) parseReturnStatement() (_ *ReturnStatement, err error) { + assert(p.peek() == RETURN) + + var stmt ReturnStatement + stmt.Return, _, _ = p.scan() + + expr, err := p.ParseExpr() + if err != nil { + return &stmt, err + } + stmt.ReturnExpr = expr + return &stmt, nil +} + func (p *Parser) parseDropFunctionStatement(dropPos Pos) (_ *DropFunctionStatement, err error) { assert(p.peek() == FUNCTION) @@ -1530,6 +1671,69 @@ func (p *Parser) parseDropFunctionStatement(dropPos Pos) (_ *DropFunctionStateme return &stmt, nil } +func (p *Parser) parseCreateModelStatement(createPos Pos) (_ *CreateModelStatement, err error) { + assert(p.peek() == MODEL) + + var stmt CreateModelStatement + stmt.Create = createPos + stmt.Model, _, _ = p.scan() + + // Parse optional "IF NOT EXISTS". + if p.peek() == IF { + stmt.If, _, _ = p.scan() + + if p.peek() != NOT { + return &stmt, p.errorExpected(p.pos, p.tok, "NOT") + } + stmt.IfNot, _, _ = p.scan() + + if p.peek() != EXISTS { + return &stmt, p.errorExpected(p.pos, p.tok, "EXISTS") + } + stmt.IfNotExists, _, _ = p.scan() + } + + if stmt.Name, err = p.parseIdent("model name"); err != nil { + return &stmt, err + } + + // options + if p.peek() != WITH { + return &stmt, p.errorExpected(p.pos, p.tok, "WITH") + } + stmt.With, _, _ = p.scan() + + stmt.Options = make([]*ModelOptionDefinition, 0) + for { + option, err := p.parseIdent("model option") + if err != nil { + return &stmt, err + } + + expr, err := p.ParseExpr() + if err != nil { + return &stmt, err + } + stmt.Options = append(stmt.Options, &ModelOptionDefinition{ + Name: option, + OptionExpr: expr, + }) + if p.peek() == AS { + break + } + } + + if p.peek() != AS { + return &stmt, p.errorExpected(p.pos, p.tok, "AS") + } + stmt.As, _, _ = p.scan() + + if stmt.ModelQuery, err = p.parseSelectStatement(false, nil); err != nil { + return &stmt, err + } + return &stmt, nil +} + func (p *Parser) parseIdent(desc string) (*Ident, error) { pos, tok, lit := p.scan() switch tok { @@ -2152,82 +2356,87 @@ func (p *Parser) parseSelectStatement(compounded bool, withClause *WithClause) ( // } //} - switch p.peek() { - /*case VALUES: - stmt.Values, _, _ = p.scan() + if p.peek() != SELECT { + return &stmt, p.errorExpected(p.pos, p.tok, "SELECT") + } - for { - var list ExprList - if p.peek() != LP { - return &stmt, p.errorExpected(p.pos, p.tok, "left paren") + stmt.Select, _, _ = p.scan() + + // Parse optional "DISTINCT". + if tok := p.peek(); tok == DISTINCT { + stmt.Distinct, _, _ = p.scan() + } + + if p.peek() == TOP { + stmt.Top, _, _ = p.scan() + if p.peek() == LP { + _, _, _ = p.scan() } - list.Lparen, _, _ = p.scan() + if stmt.TopExpr, err = p.ParseExpr(); err != nil { + return &stmt, err + } + if p.peek() == RP { + _, _, _ = p.scan() + } + } + + if p.peek() == TOPN { + stmt.TopN, _, _ = p.scan() + if p.peek() == LP { + _, _, _ = p.scan() + } + if stmt.TopExpr, err = p.ParseExpr(); err != nil { + return &stmt, err + } + if p.peek() == RP { + _, _, _ = p.scan() + } + } + + // Parse result columns. + for { + col, err := p.parseResultColumn() + if err != nil { + return &stmt, err + } + stmt.Columns = append(stmt.Columns, col) + + if p.peek() != COMMA { + break + } + p.scan() + } + + // Parse FROM clause. + if p.peek() == FROM { + stmt.From, _, _ = p.scan() + if stmt.Source, err = p.parseSource(); err != nil { + return &stmt, err + } + } + + // Parse WHERE clause. + if p.peek() == WHERE { + stmt.Where, _, _ = p.scan() + if stmt.WhereExpr, err = p.ParseExpr(); err != nil { + return &stmt, err + } + } + + // Parse GROUP BY/HAVING clause. + if p.peek() == GROUP { + stmt.Group, _, _ = p.scan() + if p.peek() != BY { + return &stmt, p.errorExpected(p.pos, p.tok, "BY") + } + stmt.GroupBy, _, _ = p.scan() for { expr, err := p.ParseExpr() if err != nil { return &stmt, err } - list.Exprs = append(list.Exprs, expr) - - if p.peek() == RP { - break - } else if p.peek() != COMMA { - return &stmt, p.errorExpected(p.pos, p.tok, "comma or right paren") - } - p.scan() - } - list.Rparen, _, _ = p.scan() - stmt.ValueLists = append(stmt.ValueLists, &list) - - if p.peek() != COMMA { - break - } - p.scan() - - }*/ - - case SELECT: - stmt.Select, _, _ = p.scan() - - // Parse optional "DISTINCT". - if tok := p.peek(); tok == DISTINCT { - stmt.Distinct, _, _ = p.scan() - } - - if p.peek() == TOP { - stmt.Top, _, _ = p.scan() - if p.peek() == LP { - _, _, _ = p.scan() - } - if stmt.TopExpr, err = p.ParseExpr(); err != nil { - return &stmt, err - } - if p.peek() == RP { - _, _, _ = p.scan() - } - } - - if p.peek() == TOPN { - stmt.TopN, _, _ = p.scan() - if p.peek() == LP { - _, _, _ = p.scan() - } - if stmt.TopExpr, err = p.ParseExpr(); err != nil { - return &stmt, err - } - if p.peek() == RP { - _, _, _ = p.scan() - } - } - - // Parse result columns. - for { - col, err := p.parseResultColumn() - if err != nil { - return &stmt, err - } - stmt.Columns = append(stmt.Columns, col) + stmt.GroupByExprs = append(stmt.GroupByExprs, expr) if p.peek() != COMMA { break @@ -2235,101 +2444,61 @@ func (p *Parser) parseSelectStatement(compounded bool, withClause *WithClause) ( p.scan() } - // Parse FROM clause. - if p.peek() == FROM { - stmt.From, _, _ = p.scan() - if stmt.Source, err = p.parseSource(); err != nil { + // Parse optional HAVING clause. + if p.peek() == HAVING { + stmt.Having, _, _ = p.scan() + if stmt.HavingExpr, err = p.ParseExpr(); err != nil { return &stmt, err } } - - // Parse WHERE clause. - if p.peek() == WHERE { - stmt.Where, _, _ = p.scan() - if stmt.WhereExpr, err = p.ParseExpr(); err != nil { - return &stmt, err - } - } - - // Parse GROUP BY/HAVING clause. - if p.peek() == GROUP { - stmt.Group, _, _ = p.scan() - if p.peek() != BY { - return &stmt, p.errorExpected(p.pos, p.tok, "BY") - } - stmt.GroupBy, _, _ = p.scan() - - for { - expr, err := p.ParseExpr() - if err != nil { - return &stmt, err - } - stmt.GroupByExprs = append(stmt.GroupByExprs, expr) - - if p.peek() != COMMA { - break - } - p.scan() - } - - // Parse optional HAVING clause. - if p.peek() == HAVING { - stmt.Having, _, _ = p.scan() - if stmt.HavingExpr, err = p.ParseExpr(); err != nil { - return &stmt, err - } - } - } - - // Parse WINDOW clause. - if p.peek() == WINDOW { - stmt.Window, _, _ = p.scan() - - for { - var window Window - if window.Name, err = p.parseIdent("window name"); err != nil { - return &stmt, err - } - - if p.peek() != AS { - return &stmt, p.errorExpected(p.pos, p.tok, "AS") - } - window.As, _, _ = p.scan() - - if window.Definition, err = p.parseWindowDefinition(); err != nil { - return &stmt, err - } - - stmt.Windows = append(stmt.Windows, &window) - - if p.peek() != COMMA { - break - } - p.scan() - } - } - default: - return &stmt, p.errorExpected(p.pos, p.tok, "SELECT") } + // Parse WINDOW clause. + // if p.peek() == WINDOW { + // stmt.Window, _, _ = p.scan() + + // for { + // var window Window + // if window.Name, err = p.parseIdent("window name"); err != nil { + // return &stmt, err + // } + + // if p.peek() != AS { + // return &stmt, p.errorExpected(p.pos, p.tok, "AS") + // } + // window.As, _, _ = p.scan() + + // if window.Definition, err = p.parseWindowDefinition(); err != nil { + // return &stmt, err + // } + + // stmt.Windows = append(stmt.Windows, &window) + + // if p.peek() != COMMA { + // break + // } + // p.scan() + // } + // } + // Optionally compound additional SELECT/VALUES. - switch tok := p.peek(); tok { - case UNION, INTERSECT, EXCEPT: - if tok == UNION { - stmt.Union, _, _ = p.scan() - if p.peek() == ALL { - stmt.UnionAll, _, _ = p.scan() - } - } else if tok == INTERSECT { - stmt.Intersect, _, _ = p.scan() - } else { - stmt.Except, _, _ = p.scan() - } + // switch tok := p.peek(); tok { + // case UNION, INTERSECT, EXCEPT: + // if tok == UNION { + // stmt.Union, _, _ = p.scan() + // if p.peek() == ALL { + // stmt.UnionAll, _, _ = p.scan() + // } + // } else if tok == INTERSECT { + // stmt.Intersect, _, _ = p.scan() + // } else { + // stmt.Except, _, _ = p.scan() + // } - if stmt.Compound, err = p.parseSelectStatement(true, nil); err != nil { - return &stmt, err - } - } + // if stmt.Compound, err = p.parseSelectStatement(true, nil); err != nil { + // return &stmt, err + // } + // } // Parse ORDER BY clause. if !compounded && p.peek() == ORDER { @@ -2353,6 +2522,13 @@ func (p *Parser) parseSelectStatement(compounded bool, withClause *WithClause) ( } } + // Parse LIMIT clause. + if !compounded && p.peek() == LIMIT { + stmt.Limit, _, _ = p.scan() + if stmt.LimitExpr, err = p.ParseExpr(); err != nil { + return &stmt, err + } + } return &stmt, nil } @@ -2704,6 +2880,26 @@ func (p *Parser) parseTableValuedFunction(ident *Ident) (_ *TableValuedFunction, return &cte, nil }*/ +func (p *Parser) parsePredictStatement() (_ *PredictStatement, err error) { + assert(p.peek() == PREDICT) + + var stmt PredictStatement + stmt.Predict, _, _ = p.scan() + if p.peek() != USING { + return &stmt, p.errorExpected(p.pos, p.tok, "USING") + } + stmt.Using, _, _ = p.scan() + + if stmt.ModelName, err = p.parseIdent("model name"); err != nil { + return &stmt, err + } + + if stmt.InputQuery, err = p.parseSelectStatement(false, nil); err != nil { + return &stmt, err + } + return &stmt, nil +} + func (p *Parser) mustParseLiteral() Expr { assert(isLiteralToken(p.tok)) pos, tok, lit := p.scan() @@ -2748,7 +2944,7 @@ func (p *Parser) parseOperand() (expr Expr, err error) { return p.parseCall(ident) } return nil, p.errorExpected(p.pos, pk, "call expression") - case STRING: + case STRING, BLOB: return &StringLit{ValuePos: pos, Value: lit}, nil case FLOAT: return &FloatLit{ValuePos: pos, Value: lit}, nil @@ -3633,6 +3829,22 @@ func isBulkInsertOptionStartToken(tok Token, p *Parser) bool { return false } +func isCopyOptionStartToken(tok Token, p *Parser) bool { + switch tok { + case IDENT: + ident, err := p.parseIdent("copy option") + defer p.unscan() + if err != nil { + return false + } + switch strings.ToUpper(ident.Name) { + case "URL", "APIKEY": + return true + } + } + return false +} + // isConstraintStartToken returns true if tok is the initial token of a constraint. func isConstraintStartToken(tok Token, isTable bool) bool { switch tok { diff --git a/sql3/parser/parser_test.go b/sql3/parser/parser_test.go index f548692b9..72b316b52 100644 --- a/sql3/parser/parser_test.go +++ b/sql3/parser/parser_test.go @@ -472,7 +472,7 @@ func TestParser_ParseAlterStatement(t *testing.T) { func TestParser_ParseFunctionStatement(t *testing.T) { t.Run("CreateFunction", func(t *testing.T) { - AssertParseStatement(t, `CREATE FUNCTION IF NOT EXISTS func (@param1 int, @param2 string) returns @scalar int as begin end`, &parser.CreateFunctionStatement{ + AssertParseStatement(t, `CREATE FUNCTION IF NOT EXISTS func (@param1 int, @param2 string) returns int as begin end`, &parser.CreateFunctionStatement{ Create: pos(0), Function: pos(7), If: pos(16), @@ -490,15 +490,12 @@ func TestParser_ParseFunctionStatement(t *testing.T) { Type: &parser.Type{Name: &parser.Ident{NamePos: pos(57), Name: "string"}}, }, }, - Rparen: pos(63), - Returns: pos(65), - ReturnDef: &parser.ParameterDefinition{ - Name: &parser.Variable{Name: "@scalar", NamePos: pos(73)}, - Type: &parser.Type{Name: &parser.Ident{NamePos: pos(81), Name: "int"}}, - }, - As: pos(85), - Begin: pos(88), - End: pos(94), + Rparen: pos(63), + Returns: pos(65), + ReturnType: &parser.Type{Name: &parser.Ident{NamePos: pos(73), Name: "int"}}, + As: pos(77), + Begin: pos(80), + End: pos(86), }) // AssertParseStatement(t, `CREATE TRIGGER IF NOT EXISTS trig BEFORE INSERT ON tbl BEGIN DELETE FROM new; END`, &parser.CreateFunctionStatement{ // Create: pos(0), @@ -955,7 +952,7 @@ func TestParser_ParseStatement(t *testing.T) { }, }) - AssertParseStatementError(t, `CREATE`, `1:1: expected DATABASE, TABLE, VIEW or FUNCTION`) + AssertParseStatementError(t, `CREATE`, `1:1: expected DATABASE, TABLE, VIEW, FUNCTION or MODEL`) AssertParseStatementError(t, `CREATE DATABASE`, `1:15: expected database name, found 'EOF'`) AssertParseStatementError(t, `CREATE DATABASE IF`, `1:18: expected NOT, found 'EOF'`) AssertParseStatementError(t, `CREATE DATABASE IF NOT`, `1:22: expected EXISTS, found 'EOF'`) @@ -2215,29 +2212,29 @@ func TestParser_ParseStatement(t *testing.T) { Having: pos(22), HavingExpr: &parser.BoolLit{ValuePos: pos(29), Value: true}, }) - AssertParseStatement(t, `SELECT * WINDOW win1 AS (), win2 AS ()`, &parser.SelectStatement{ - Select: pos(0), - Columns: []*parser.ResultColumn{{Star: pos(7)}}, - Window: pos(9), - Windows: []*parser.Window{ - { - Name: &parser.Ident{NamePos: pos(16), Name: "win1"}, - As: pos(21), - Definition: &parser.WindowDefinition{ - Lparen: pos(24), - Rparen: pos(25), - }, - }, - { - Name: &parser.Ident{NamePos: pos(28), Name: "win2"}, - As: pos(33), - Definition: &parser.WindowDefinition{ - Lparen: pos(36), - Rparen: pos(37), - }, - }, - }, - }) + // AssertParseStatement(t, `SELECT * WINDOW win1 AS (), win2 AS ()`, &parser.SelectStatement{ + // Select: pos(0), + // Columns: []*parser.ResultColumn{{Star: pos(7)}}, + // Window: pos(9), + // Windows: []*parser.Window{ + // { + // Name: &parser.Ident{NamePos: pos(16), Name: "win1"}, + // As: pos(21), + // Definition: &parser.WindowDefinition{ + // Lparen: pos(24), + // Rparen: pos(25), + // }, + // }, + // { + // Name: &parser.Ident{NamePos: pos(28), Name: "win2"}, + // As: pos(33), + // Definition: &parser.WindowDefinition{ + // Lparen: pos(36), + // Rparen: pos(37), + // }, + // }, + // }, + // }) AssertParseStatement(t, `SELECT * ORDER BY foo ASC, bar DESC`, &parser.SelectStatement{ Select: pos(0), @@ -2252,64 +2249,64 @@ func TestParser_ParseStatement(t *testing.T) { }, }) - AssertParseStatement(t, `SELECT * UNION SELECT * ORDER BY foo`, &parser.SelectStatement{ - Select: pos(0), - Columns: []*parser.ResultColumn{ - {Star: pos(7)}, - }, - Union: pos(9), - Compound: &parser.SelectStatement{ - Select: pos(15), - Columns: []*parser.ResultColumn{ - {Star: pos(22)}, - }, - }, - Order: pos(24), - OrderBy: pos(30), - OrderingTerms: []*parser.OrderingTerm{ - {X: &parser.Ident{NamePos: pos(33), Name: "foo"}}, - }, - }) - AssertParseStatement(t, `SELECT * UNION ALL SELECT *`, &parser.SelectStatement{ - Select: pos(0), - Columns: []*parser.ResultColumn{ - {Star: pos(7)}, - }, - Union: pos(9), - UnionAll: pos(15), - Compound: &parser.SelectStatement{ - Select: pos(19), - Columns: []*parser.ResultColumn{ - {Star: pos(26)}, - }, - }, - }) - AssertParseStatement(t, `SELECT * INTERSECT SELECT *`, &parser.SelectStatement{ - Select: pos(0), - Columns: []*parser.ResultColumn{ - {Star: pos(7)}, - }, - Intersect: pos(9), - Compound: &parser.SelectStatement{ - Select: pos(19), - Columns: []*parser.ResultColumn{ - {Star: pos(26)}, - }, - }, - }) - AssertParseStatement(t, `SELECT * EXCEPT SELECT *`, &parser.SelectStatement{ - Select: pos(0), - Columns: []*parser.ResultColumn{ - {Star: pos(7)}, - }, - Except: pos(9), - Compound: &parser.SelectStatement{ - Select: pos(16), - Columns: []*parser.ResultColumn{ - {Star: pos(23)}, - }, - }, - }) + // AssertParseStatement(t, `SELECT * UNION SELECT * ORDER BY foo`, &parser.SelectStatement{ + // Select: pos(0), + // Columns: []*parser.ResultColumn{ + // {Star: pos(7)}, + // }, + // Union: pos(9), + // Compound: &parser.SelectStatement{ + // Select: pos(15), + // Columns: []*parser.ResultColumn{ + // {Star: pos(22)}, + // }, + // }, + // Order: pos(24), + // OrderBy: pos(30), + // OrderingTerms: []*parser.OrderingTerm{ + // {X: &parser.Ident{NamePos: pos(33), Name: "foo"}}, + // }, + // }) + // AssertParseStatement(t, `SELECT * UNION ALL SELECT *`, &parser.SelectStatement{ + // Select: pos(0), + // Columns: []*parser.ResultColumn{ + // {Star: pos(7)}, + // }, + // Union: pos(9), + // UnionAll: pos(15), + // Compound: &parser.SelectStatement{ + // Select: pos(19), + // Columns: []*parser.ResultColumn{ + // {Star: pos(26)}, + // }, + // }, + // }) + // AssertParseStatement(t, `SELECT * INTERSECT SELECT *`, &parser.SelectStatement{ + // Select: pos(0), + // Columns: []*parser.ResultColumn{ + // {Star: pos(7)}, + // }, + // Intersect: pos(9), + // Compound: &parser.SelectStatement{ + // Select: pos(19), + // Columns: []*parser.ResultColumn{ + // {Star: pos(26)}, + // }, + // }, + // }) + // AssertParseStatement(t, `SELECT * EXCEPT SELECT *`, &parser.SelectStatement{ + // Select: pos(0), + // Columns: []*parser.ResultColumn{ + // {Star: pos(7)}, + // }, + // Except: pos(9), + // Compound: &parser.SelectStatement{ + // Select: pos(16), + // Columns: []*parser.ResultColumn{ + // {Star: pos(23)}, + // }, + // }, + // }) /*AssertParseStatement(t, `VALUES (1, 2), (3, 4)`, &parser.SelectStatement{ Values: pos(0), @@ -3402,11 +3399,11 @@ func TestParser_ParseStatement(t *testing.T) { AssertParseStatementError(t, `SELECT * GROUP BY`, `1:17: expected expression, found 'EOF'`) AssertParseStatementError(t, `SELECT * GROUP BY foo bar`, `1:23: expected semicolon or EOF, found bar`) AssertParseStatementError(t, `SELECT * GROUP BY foo HAVING`, `1:28: expected expression, found 'EOF'`) - AssertParseStatementError(t, `SELECT * WINDOW`, `1:15: expected window name, found 'EOF'`) - AssertParseStatementError(t, `SELECT * WINDOW win1`, `1:20: expected AS, found 'EOF'`) - AssertParseStatementError(t, `SELECT * WINDOW win1 AS`, `1:23: expected left paren, found 'EOF'`) - AssertParseStatementError(t, `SELECT * WINDOW win1 AS (`, `1:25: expected right paren, found 'EOF'`) - AssertParseStatementError(t, `SELECT * WINDOW win1 AS () win2`, `1:28: expected semicolon or EOF, found win2`) + // AssertParseStatementError(t, `SELECT * WINDOW`, `1:15: expected window name, found 'EOF'`) + // AssertParseStatementError(t, `SELECT * WINDOW win1`, `1:20: expected AS, found 'EOF'`) + // AssertParseStatementError(t, `SELECT * WINDOW win1 AS`, `1:23: expected left paren, found 'EOF'`) + // AssertParseStatementError(t, `SELECT * WINDOW win1 AS (`, `1:25: expected right paren, found 'EOF'`) + // AssertParseStatementError(t, `SELECT * WINDOW win1 AS () win2`, `1:28: expected semicolon or EOF, found win2`) AssertParseStatementError(t, `SELECT * ORDER`, `1:14: expected BY, found 'EOF'`) AssertParseStatementError(t, `SELECT * ORDER BY`, `1:17: expected expression, found 'EOF'`) AssertParseStatementError(t, `SELECT * ORDER BY 1,`, `1:20: expected expression, found 'EOF'`) diff --git a/sql3/parser/token.go b/sql3/parser/token.go index ef16f7ef2..165e07243 100644 --- a/sql3/parser/token.go +++ b/sql3/parser/token.go @@ -78,8 +78,6 @@ const ( ACTION ADD AFTER - AGG_COLUMN - AGG_FUNCTION ALL ALTER ANALYZE @@ -106,9 +104,9 @@ const ( COMMENT CONFLICT CONSTRAINT + COPY CREATE CROSS - CTIME_KW CURRENT CURRENT_DATE CURRENT_TIMESTAMP @@ -167,11 +165,13 @@ const ( LAST LEFT LIKE + LIMIT LRU MAP MATCH MAX MIN + MODEL NO NOT NOTBETWEEN @@ -194,6 +194,7 @@ const ( PLAN PRAGMA PRECEDING + PREDICT PRIMARY QUERY RANGE @@ -304,8 +305,6 @@ var tokens = [...]string{ ACTION: "ACTION", ADD: "ADD", AFTER: "AFTER", - AGG_COLUMN: "AGG_COLUMN", - AGG_FUNCTION: "AGG_FUNCTION", ALL: "ALL", ALTER: "ALTER", ANALYZE: "ANALYZE", @@ -332,9 +331,9 @@ var tokens = [...]string{ COMMENT: "COMMENT", CONFLICT: "CONFLICT", CONSTRAINT: "CONSTRAINT", + COPY: "COPY", CREATE: "CREATE", CROSS: "CROSS", - CTIME_KW: "CTIME_KW", CURRENT: "CURRENT", CURRENT_DATE: "CURRENT_DATE", CURRENT_TIMESTAMP: "CURRENT_TIMESTAMP", @@ -393,11 +392,13 @@ var tokens = [...]string{ LAST: "LAST", LEFT: "LEFT", LIKE: "LIKE", + LIMIT: "LIMIT", MAP: "MAP", LRU: "LRU", MATCH: "MATCH", MAX: "MAX", MIN: "MIN", + MODEL: "MODEL", NO: "NO", NOT: "NOT", NOTBETWEEN: "NOTBETWEEN", @@ -420,6 +421,7 @@ var tokens = [...]string{ PLAN: "PLAN", PRAGMA: "PRAGMA", PRECEDING: "PRECEDING", + PREDICT: "PREDICT", PRIMARY: "PRIMARY", QUERY: "QUERY", RANGE: "RANGE", diff --git a/sql3/planner/compilecopy.go b/sql3/planner/compilecopy.go new file mode 100644 index 000000000..79d8c78bc --- /dev/null +++ b/sql3/planner/compilecopy.go @@ -0,0 +1,120 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +// compileCopyStatement compiles a parser.CopyStatement AST into a PlanOperator +func (p *ExecutionPlanner) compileCopyStatement(stmt *parser.CopyStatement) (types.PlanOperator, error) { + query := NewPlanOpQuery(p, NewPlanOpNullTable(), p.sql) + query.AddWarning("🦖 here there be dragons! COPY statement is experimental.") + + // handle projections + projections := make([]types.PlanExpression, 0) + for _, c := range stmt.Source.PossibleOutputColumns() { + expr := &parser.QualifiedRef{ + Table: &parser.Ident{Name: c.TableName}, + Column: &parser.Ident{Name: c.ColumnName}, + ColumnIndex: c.ColumnIndex, + RefDataType: c.Datatype, + } + planExpr, err := p.compileExpr(expr) + if err != nil { + return nil, err + } + projections = append(projections, planExpr) + } + + // handle the where clause + where, err := p.compileExpr(stmt.WhereExpr) + if err != nil { + return nil, err + } + + // compile source + source, err := p.compileSource(query, stmt.Source) + if err != nil { + return nil, err + } + + // if we did have a where, insert the filter op + if where != nil { + source = NewPlanOpFilter(p, where, source) + } + + var compiledOp types.PlanOperator + url := "" + apiKey := "" + + if stmt.Url != nil { + lit, ok := stmt.Url.(*parser.StringLit) + if !ok { + return nil, sql3.NewErrStringLiteral(stmt.Url.Pos().Line, stmt.Url.Pos().Column) + } + url = lit.Value + } + + if stmt.ApiKey != nil { + lit, ok := stmt.ApiKey.(*parser.StringLit) + if !ok { + return nil, sql3.NewErrStringLiteral(stmt.ApiKey.Pos().Line, stmt.ApiKey.Pos().Column) + } + apiKey = lit.Value + } + + // get the source table + tname := dax.TableName(stmt.Source.String()) + tbl, err := p.schemaAPI.TableByName(context.Background(), tname) + if err != nil { + if isTableNotFoundError(err) { + return nil, sql3.NewErrTableNotFound(0, 0, stmt.Source.String()) + } + return nil, err + } + // get the ddl of source table and subst target table name + ddl := generateTableDDL(tbl, stmt.TargetName.Name) + compiledOp = NewPlanOpCopy(p, stmt.TargetName.Name, url, apiKey, ddl, NewPlanOpProjection(projections, source)) + children := []types.PlanOperator{ + compiledOp, + } + return query.WithChildren(children...) +} + +func (p *ExecutionPlanner) analyzeCopyStatement(ctx context.Context, stmt *parser.CopyStatement) error { + + // analyze source + var err error + source, err := p.analyzeSource(ctx, stmt.Source, stmt) + if err != nil { + return err + } + stmt.Source = source + + // analyze where + expr, err := p.analyzeExpression(ctx, stmt.WhereExpr, stmt) + if err != nil { + return err + } + stmt.WhereExpr = expr + + expr, err = p.analyzeExpression(ctx, stmt.Url, stmt) + if err != nil { + return err + } + stmt.Url = expr + + expr, err = p.analyzeExpression(ctx, stmt.ApiKey, stmt) + if err != nil { + return err + } + stmt.ApiKey = expr + + return nil +} diff --git a/sql3/planner/compilecreatefunction.go b/sql3/planner/compilecreatefunction.go new file mode 100644 index 000000000..fbb055b7f --- /dev/null +++ b/sql3/planner/compilecreatefunction.go @@ -0,0 +1,79 @@ +// Copyright 2023 Molecula Corp. All rights reserved. + +package planner + +import ( + "strings" + + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +// compileCreateFunctionStatement compiles a parser.CreateFunctionStatement AST into a PlanOperator +func (p *ExecutionPlanner) compileCreateFunctionStatement(stmt *parser.CreateFunctionStatement) (types.PlanOperator, error) { + functionName := parser.IdentName(stmt.Name) + function := &functionSystemObject{ + name: functionName, + } + + lang := "sql" + if len(stmt.Options) > 0 { + for _, o := range stmt.Options { + switch strings.ToLower(o.Name.String()) { + case "language": + lit, ok := o.OptionExpr.(*parser.StringLit) + if !ok { + return nil, sql3.NewErrStringLiteral(o.OptionExpr.Pos().Line, o.OptionExpr.Pos().Column) + } + l := strings.ToLower(lit.Value) + switch l { + case "python": + lang = l + default: + return nil, sql3.NewErrInternalf("unsupported language '%s'", l) + } + } + } + } + function.language = lang + + // TODO(pok) - hobble user defined functions for now + + switch lang { + case "sql": + return nil, sql3.NewErrInternalf("unsupported language '%s'", lang) + case "python": + // return nil, sql3.NewErrInternalf("unsupported language '%s'", lang) + + // function body is in the return statement + if len(stmt.Body) != 1 { + return nil, sql3.NewErrInternalf("unexpected body len '%d'", len(stmt.Body)) + } + + rs, ok := stmt.Body[0].(*parser.ReturnStatement) + if !ok { + return nil, sql3.NewErrInternalf("unexpected statement type '%T'", stmt.Body[0]) + } + + bexpr, ok := rs.ReturnExpr.(*parser.StringLit) + if !ok { + return nil, sql3.NewErrInternalf("unexpected expression type '%T'", rs.ReturnExpr) + } + + function.body = bexpr.Value + default: + return nil, sql3.NewErrInternalf("unsupported language '%s'", lang) + } + + fn := NewPlanOpCreateFunction(p, stmt.IfNotExists.IsValid(), function) + fn.AddWarning("🦖 here there be dragons! CREATE FUNCTION statement is experimental.") + + query := NewPlanOpQuery(p, fn, p.sql) + return query, nil +} + +func (p *ExecutionPlanner) analyzeCreateFunctionStatement(stmt *parser.CreateFunctionStatement) error { + + return nil +} diff --git a/sql3/planner/compilecreatemodel.go b/sql3/planner/compilecreatemodel.go new file mode 100644 index 000000000..9cd57a056 --- /dev/null +++ b/sql3/planner/compilecreatemodel.go @@ -0,0 +1,180 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "strings" + + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +// TODO (pok) what does 'if not exists' do? + +// compileCreateModelStatement compiles a parser.CreateModelStatement AST into a PlanOperator +func (p *ExecutionPlanner) compileCreateModelStatement(stmt *parser.CreateModelStatement) (types.PlanOperator, error) { + modelName := parser.IdentName(stmt.Name) + + // does the model exist + obj, err := p.getModelByName(modelName) + if err != nil { + return nil, err + } + if obj != nil { + return nil, sql3.NewErrInternalf("model '%s' already exists", modelName) + } + + // if we got to here model does not exist + model := &modelSystemObject{ + name: modelName, + } + + for _, o := range stmt.Options { + optName := parser.IdentName(o.Name) + + switch strings.ToLower(optName) { + case "modeltype": + lit, ok := o.OptionExpr.(*parser.StringLit) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", o.OptionExpr) + } + model.modelType = lit.Value + + case "labels": + lit, ok := o.OptionExpr.(*parser.SetLiteralExpr) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", o.OptionExpr) + } + model.labels = make([]string, len(lit.Members)) + for i, m := range lit.Members { + mlit, ok := m.(*parser.StringLit) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", m) + } + model.labels[i] = mlit.Value + } + + default: + return nil, sql3.NewErrInternalf("unexpected model option '%s'", optName) + } + } + + selOp, err := p.compileSelectStatement(stmt.ModelQuery, true) + if err != nil { + return nil, err + } + + // build a list of input columns for the model from the select query + schema := selOp.Schema() + model.inputColumns = make([]string, 0) + for _, p := range schema { + // if we have no column name, we have an error + if len(p.ColumnName) == 0 { + return nil, sql3.NewErrInternalf("query output columns used as inputs to models must be named") + } + // exclude any that are in the labels + isLabel := false + for _, l := range model.labels { + if strings.EqualFold(p.ColumnName, l) { + isLabel = true + break + } + } + if !isLabel { + model.inputColumns = append(model.inputColumns, p.ColumnName) + } + } + createModel := NewPlanOpCreateModel(p, model, selOp) + createModel.AddWarning("🦖 here there be dragons! CREATE MODEL statement is experimental.") + + query := NewPlanOpQuery(p, createModel, p.sql) + return query, nil +} + +func (p *ExecutionPlanner) analyzeCreateModelStatement(ctx context.Context, stmt *parser.CreateModelStatement) error { + // iterate the options + for _, opt := range stmt.Options { + optName := parser.IdentName(opt.Name) + if !isValidModelOption(optName) { + return sql3.NewErrInternalf("invalid model option '%s'", optName) + } + e, err := p.analyzeModelOptionExpr(ctx, optName, opt.OptionExpr, stmt) + if err != nil { + return err + } + opt.OptionExpr = e + } + + // analyze the select + _, err := p.analyzeSelectStatement(ctx, stmt.ModelQuery) + if err != nil { + return err + } + return nil +} + +func isValidModelOption(name string) bool { + switch strings.ToLower(name) { + case "modeltype": + return true + + case "labels": + return true + + default: + return false + } +} + +func (p *ExecutionPlanner) analyzeModelOptionExpr(ctx context.Context, optName string, expr parser.Expr, scope parser.Statement) (parser.Expr, error) { + if expr == nil { + return nil, nil + } + + e, err := p.analyzeExpression(ctx, expr, scope) + if err != nil { + return nil, err + } + + switch strings.ToLower(optName) { + case "modeltype": + + // model type needs to be a string literal + if !(e.IsLiteral() && typeIsString(e.DataType())) { + return nil, sql3.NewErrStringLiteral(e.Pos().Line, e.Pos().Column) + } + ty, ok := e.(*parser.StringLit) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", e) + } + + // these are the model types supported + switch strings.ToLower(ty.Value) { + case "linear_regresssion": + break + default: + return nil, sql3.NewErrInternalf("unexpected model tyoe '%s'", ty.Value) + } + return e, nil + + case "labels": + // labels needs to be a string array literal + // TODO (pok) revist 'set' literals (should be array literal; type checking could be robustified etc.) + if !e.IsLiteral() { + return nil, sql3.NewErrInternalf("string array literal expected") + } + ok, baseType := typeIsSet(e.DataType()) + if !ok { + return nil, sql3.NewErrInternalf("array expression expected") + } + if !typeIsString(baseType) { + return nil, sql3.NewErrInternalf("string array expected") + } + return e, nil + + default: + return nil, sql3.NewErrInternalf("unexpected option name '%s'", optName) + } +} diff --git a/sql3/planner/compiledropmodel.go b/sql3/planner/compiledropmodel.go new file mode 100644 index 000000000..e9d221230 --- /dev/null +++ b/sql3/planner/compiledropmodel.go @@ -0,0 +1,26 @@ +// Copyright 2023 Molecula Corp. All rights reserved. + +package planner + +import ( + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +// compileDropModelStatement compiles a DROP MODEL statement into a PlanOperator. +func (p *ExecutionPlanner) compileDropModelStatement(stmt *parser.DropModelStatement) (_ types.PlanOperator, err error) { + modelName := parser.IdentName(stmt.Name) + v, err := p.getModelByName(modelName) + if err != nil { + return nil, err + } + if v == nil && !stmt.IfExists.IsValid() { + return nil, sql3.NewErrModelNotFound(0, 0, modelName) + } + + dropModel := NewPlanOpDropModel(p, stmt.IfExists.IsValid(), modelName) + dropModel.AddWarning("🦖 here there be dragons! DROP MODEL statement is experimental.") + + return NewPlanOpQuery(p, dropModel, p.sql), nil +} diff --git a/sql3/planner/compilepredict.go b/sql3/planner/compilepredict.go new file mode 100644 index 000000000..5219ddcea --- /dev/null +++ b/sql3/planner/compilepredict.go @@ -0,0 +1,50 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +// compilePredictStatement compiles a parser.PredictStatement AST into a PlanOperator +func (p *ExecutionPlanner) compilePredictStatement(ctx context.Context, stmt *parser.PredictStatement) (types.PlanOperator, error) { + + // go get the model + modelName := parser.IdentName(stmt.ModelName) + + // does the model exist + obj, err := p.getModelByName(modelName) + if err != nil { + return nil, err + } + if obj == nil { + return nil, sql3.NewErrInternalf("model '%s' not found", modelName) + } + + selOp, err := p.compileSelectStatement(stmt.InputQuery, true) + if err != nil { + return nil, err + } + + predict := NewPlanOpPredict(p, obj, selOp) + predict.AddWarning("🦖 here there be dragons! PREDICT statement is experimental.") + + query := NewPlanOpQuery(p, predict, p.sql) + + return query, nil +} + +func (p *ExecutionPlanner) analyzePredictStatement(ctx context.Context, stmt *parser.PredictStatement) error { + + // analyze the select + _, err := p.analyzeSelectStatement(ctx, stmt.InputQuery) + if err != nil { + return err + } + + return nil +} diff --git a/sql3/planner/compileselect.go b/sql3/planner/compileselect.go index 2c986d3bb..c8e076c10 100644 --- a/sql3/planner/compileselect.go +++ b/sql3/planner/compileselect.go @@ -305,7 +305,7 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, } } - // insert the top operator if it exists + // insert the top operator if it exists, or limit - analyzer should have caught the case of both existing if stmt.Top.IsValid() { topExpr, err := p.compileExpr(stmt.TopExpr) if err != nil { @@ -313,6 +313,14 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement, } compiledOp = NewPlanOpTop(topExpr, compiledOp) } + // handle limit + if stmt.Limit.IsValid() { + limitExpr, err := p.compileExpr(stmt.LimitExpr) + if err != nil { + return nil, err + } + compiledOp = NewPlanOpTop(limitExpr, compiledOp) + } // handle distinct if stmt.Distinct.IsValid() { @@ -613,6 +621,10 @@ func (p *ExecutionPlanner) analyzeSelectStatement(ctx context.Context, stmt *par } } + if stmt.TopExpr != nil && stmt.LimitExpr != nil { + return nil, sql3.NewErrErrTopLimitCannotCoexist(stmt.TopExpr.Pos().Line, stmt.TopExpr.Pos().Column) + } + expr, err := p.analyzeExpression(ctx, stmt.TopExpr, stmt) if err != nil { return nil, err @@ -624,6 +636,17 @@ func (p *ExecutionPlanner) analyzeSelectStatement(ctx context.Context, stmt *par stmt.TopExpr = expr } + expr, err = p.analyzeExpression(ctx, stmt.LimitExpr, stmt) + if err != nil { + return nil, err + } + if expr != nil { + if !(expr.IsLiteral() && typeIsInteger(expr.DataType())) { + return nil, sql3.NewErrIntegerLiteral(stmt.LimitExpr.Pos().Line, stmt.LimitExpr.Pos().Column) + } + stmt.LimitExpr = expr + } + expr, err = p.analyzeExpression(ctx, stmt.HavingExpr, stmt) if err != nil { return nil, err diff --git a/sql3/planner/executionplanner.go b/sql3/planner/executionplanner.go index 4f607173b..53cf69848 100644 --- a/sql3/planner/executionplanner.go +++ b/sql3/planner/executionplanner.go @@ -69,6 +69,10 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen rootOperator, err = p.compileSelectStatement(stmt, false) case *parser.ShowDatabasesStatement: rootOperator, err = p.compileShowDatabasesStatement(ctx, stmt) + case *parser.CopyStatement: + rootOperator, err = p.compileCopyStatement(stmt) + case *parser.PredictStatement: + rootOperator, err = p.compilePredictStatement(ctx, stmt) case *parser.ShowTablesStatement: rootOperator, err = p.compileShowTablesStatement(ctx, stmt) case *parser.ShowColumnsStatement: @@ -93,16 +97,23 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen rootOperator, err = p.compileDropTableStatement(ctx, stmt) case *parser.DropViewStatement: rootOperator, err = p.compileDropViewStatement(ctx, stmt) + case *parser.DropModelStatement: + rootOperator, err = p.compileDropModelStatement(stmt) case *parser.InsertStatement: rootOperator, err = p.compileInsertStatement(ctx, stmt) case *parser.BulkInsertStatement: rootOperator, err = p.compileBulkInsertStatement(ctx, stmt) case *parser.DeleteStatement: rootOperator, err = p.compileDeleteStatement(stmt) + case *parser.CreateModelStatement: + rootOperator, err = p.compileCreateModelStatement(stmt) + case *parser.CreateFunctionStatement: + rootOperator, err = p.compileCreateFunctionStatement(stmt) + default: return nil, sql3.NewErrInternalf("cannot plan statement: %T", stmt) } - // Optimize the plan. + // optimize the plan if err == nil { rootOperator, err = p.optimizePlan(ctx, rootOperator) } @@ -130,6 +141,10 @@ func (p *ExecutionPlanner) analyzePlan(ctx context.Context, stmt parser.Statemen return err case *parser.ShowDatabasesStatement: return nil + case *parser.CopyStatement: + return p.analyzeCopyStatement(ctx, stmt) + case *parser.PredictStatement: + return p.analyzePredictStatement(ctx, stmt) case *parser.ShowTablesStatement: return nil case *parser.ShowColumnsStatement: @@ -154,12 +169,19 @@ func (p *ExecutionPlanner) analyzePlan(ctx context.Context, stmt parser.Statemen return nil case *parser.DropViewStatement: return nil + case *parser.DropModelStatement: + return nil case *parser.InsertStatement: return p.analyzeInsertStatement(ctx, stmt) case *parser.BulkInsertStatement: return p.analyzeBulkInsertStatement(ctx, stmt) case *parser.DeleteStatement: return p.analyzeDeleteStatement(ctx, stmt) + case *parser.CreateModelStatement: + return p.analyzeCreateModelStatement(ctx, stmt) + case *parser.CreateFunctionStatement: + return p.analyzeCreateFunctionStatement(stmt) + default: return sql3.NewErrInternalf("cannot analyze statement: %T", stmt) } diff --git a/sql3/planner/expression.go b/sql3/planner/expression.go index 260e7052d..717f5486c 100644 --- a/sql3/planner/expression.go +++ b/sql3/planner/expression.go @@ -1514,16 +1514,18 @@ func (n *inOpPlanExpression) WithChildren(children ...types.PlanExpression) (typ // callPlanExpression is a function call type callPlanExpression struct { - name string - args []types.PlanExpression - dataType parser.ExprDataType + name string + args []types.PlanExpression + dataType parser.ExprDataType + udfReference *functionSystemObject } -func newCallPlanExpression(name string, args []types.PlanExpression, dataType parser.ExprDataType) *callPlanExpression { +func newCallPlanExpression(name string, args []types.PlanExpression, dataType parser.ExprDataType, udfReference *functionSystemObject) *callPlanExpression { return &callPlanExpression{ - name: name, - args: args, - dataType: dataType, + name: name, + args: args, + dataType: dataType, + udfReference: udfReference, } } @@ -1591,6 +1593,9 @@ func (n *callPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er case "DATETIMEDIFF": return n.EvaluateDatetimeDiff(currentRow) default: + if n.udfReference != nil { + return n.evaluateUserDefinedFunction(currentRow) + } return nil, sql3.NewErrInternalf("unhandled function name '%s'", n.name) } } @@ -1632,7 +1637,7 @@ func (n *callPlanExpression) WithChildren(children ...types.PlanExpression) (typ if len(children) != len(n.args) { return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) } - return newCallPlanExpression(n.name, children, n.dataType), nil + return newCallPlanExpression(n.name, children, n.dataType, n.udfReference), nil } // aliasPlanExpression is a alias ref @@ -2049,30 +2054,30 @@ func (n *sysVariablePlanExpression) WithChildren(children ...types.PlanExpressio return n, nil } -// dateLiteralPlanExpression is a date literal -type dateLiteralPlanExpression struct { +// timestampLiteralPlanExpression is a date literal +type timestampLiteralPlanExpression struct { value time.Time } -func newDateLiteralPlanExpression(value time.Time) *dateLiteralPlanExpression { - return &dateLiteralPlanExpression{ +func newTimestampLiteralPlanExpression(value time.Time) *timestampLiteralPlanExpression { + return ×tampLiteralPlanExpression{ value: value, } } -func (n *dateLiteralPlanExpression) Evaluate(currentRow []interface{}) (interface{}, error) { +func (n *timestampLiteralPlanExpression) Evaluate(currentRow []interface{}) (interface{}, error) { return n.value, nil } -func (n *dateLiteralPlanExpression) Type() parser.ExprDataType { +func (n *timestampLiteralPlanExpression) Type() parser.ExprDataType { return parser.NewDataTypeTimestamp() } -func (n *dateLiteralPlanExpression) String() string { +func (n *timestampLiteralPlanExpression) String() string { return n.value.Format(time.RFC3339Nano) } -func (n *dateLiteralPlanExpression) Plan() map[string]interface{} { +func (n *timestampLiteralPlanExpression) Plan() map[string]interface{} { result := make(map[string]interface{}) result["_expr"] = fmt.Sprintf("%T", n) result["description"] = n.String() @@ -2081,11 +2086,11 @@ func (n *dateLiteralPlanExpression) Plan() map[string]interface{} { return result } -func (n *dateLiteralPlanExpression) Children() []types.PlanExpression { +func (n *timestampLiteralPlanExpression) Children() []types.PlanExpression { return []types.PlanExpression{} } -func (n *dateLiteralPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) { +func (n *timestampLiteralPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) { return n, nil } @@ -2660,7 +2665,7 @@ func (p *ExecutionPlanner) compileExpr(expr parser.Expr) (_ types.PlanExpression return newFloatLiteralPlanExpression(expr.Value), nil case *parser.DateLit: - return newDateLiteralPlanExpression(expr.Value), nil + return newTimestampLiteralPlanExpression(expr.Value), nil case *parser.SysVariable: return newSysVariablePlanExpression(expr.Name(), expr.Token), nil @@ -2892,6 +2897,14 @@ func (p *ExecutionPlanner) compileCallExpr(expr *parser.Call) (_ types.PlanExpre agg := newPercentilePlanExpression(args[0], args[1], expr.ResultDataType) return agg, nil + case "CORR": + agg := newCorrPlanExpression(args[0], args[1], expr.ResultDataType) + return agg, nil + + case "VAR": + agg := newVarPlanExpression(args[0], expr.ResultDataType) + return agg, nil + case "MIN": agg := newMinPlanExpression(args[0], expr.ResultDataType) return agg, nil @@ -2901,7 +2914,12 @@ func (p *ExecutionPlanner) compileCallExpr(expr *parser.Call) (_ types.PlanExpre return agg, nil default: - return newCallPlanExpression(parser.IdentName(expr.Name), args, expr.ResultDataType), nil + // could be a udf - try to look it up in functions + fn, err := p.getFunctionByName(strings.ToLower(callName)) + if err != nil { + return nil, err + } + return newCallPlanExpression(parser.IdentName(expr.Name), args, expr.ResultDataType, fn), nil } } diff --git a/sql3/planner/expression_it_test.go b/sql3/planner/expression_it_test.go index bf23baa34..d9f8d870b 100644 --- a/sql3/planner/expression_it_test.go +++ b/sql3/planner/expression_it_test.go @@ -40,7 +40,7 @@ func TestExpressions(t *testing.T) { iop := newInOpPlanExpression(newIntLiteralPlanExpression(10), parser.IN, newIntLiteralPlanExpression(20)) assert.Equal(t, iop.String(), "10 in (20)") - callop := newCallPlanExpression("foo", []types.PlanExpression{newIntLiteralPlanExpression(10)}, parser.NewDataTypeInt()) + callop := newCallPlanExpression("foo", []types.PlanExpression{newIntLiteralPlanExpression(10)}, parser.NewDataTypeInt(), nil) assert.Equal(t, callop.String(), "foo(10)") alop := newAliasPlanExpression("frobny", newIntLiteralPlanExpression(10)) @@ -65,7 +65,7 @@ func TestExpressions(t *testing.T) { assert.Equal(t, blop.String(), "false") tm, _ := time.ParseInLocation(time.RFC3339, "2012-11-01T22:08:41+00:00", time.UTC) - dlop := newDateLiteralPlanExpression(tm) + dlop := newTimestampLiteralPlanExpression(tm) assert.Equal(t, dlop.String(), "2012-11-01T22:08:41Z") slop := newStringLiteralPlanExpression("foo") diff --git a/sql3/planner/expressionagg.go b/sql3/planner/expressionagg.go index eab7f098d..b260a0c45 100644 --- a/sql3/planner/expressionagg.go +++ b/sql3/planner/expressionagg.go @@ -5,6 +5,7 @@ package planner import ( "context" "fmt" + "math" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/sql3" @@ -943,6 +944,311 @@ func (n *percentilePlanExpression) WithChildren(children ...types.PlanExpression return newPercentilePlanExpression(children[0], children[1], n.returnDataType), nil } +// aggregator for CORR() +type aggregateCorr struct { + expr *corrPlanExpression + + n int64 + sum_X float64 + sum_Y float64 + sum_XY float64 + squareSum_X float64 + squareSum_Y float64 +} + +func NewAggCorrBuffer(child *corrPlanExpression) *aggregateCorr { + return &aggregateCorr{ + expr: child, + } +} + +func (m *aggregateCorr) Update(ctx context.Context, row types.Row) error { + v1, err := m.expr.arg1.Evaluate(row) + if err != nil { + return err + } + + v2, err := m.expr.arg2.Evaluate(row) + if err != nil { + return err + } + + // skip if nil + if v1 == nil || v2 == nil { + return nil + } + + var xVal float64 + var yVal float64 + + switch dataType := m.expr.arg1.Type().(type) { + case *parser.DataTypeDecimal: + thisVal, ok := v1.(pql.Decimal) + if !ok { + return sql3.NewErrInternalf("unexpected type conversion '%T'", v1) + } + + xVal = thisVal.Float64() + + case *parser.DataTypeInt: + thisVal, ok := v1.(int64) + if !ok { + return sql3.NewErrInternalf("unexpected type conversion '%T'", v1) + } + + xVal = float64(thisVal) + + default: + return sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", dataType) + } + + switch dataType := m.expr.arg2.Type().(type) { + case *parser.DataTypeDecimal: + thisVal, ok := v2.(pql.Decimal) + if !ok { + return sql3.NewErrInternalf("unexpected type conversion '%T'", v2) + } + + yVal = thisVal.Float64() + + case *parser.DataTypeInt: + thisVal, ok := v2.(int64) + if !ok { + return sql3.NewErrInternalf("unexpected type conversion '%T'", v2) + } + yVal = float64(thisVal) + + default: + return sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", dataType) + } + + m.sum_X = m.sum_X + xVal + m.sum_Y = m.sum_Y + yVal + + m.sum_XY = m.sum_XY + xVal*yVal + + m.squareSum_X = m.squareSum_X + xVal*xVal + m.squareSum_Y = m.squareSum_Y + yVal*yVal + m.n += 1 + + return nil +} + +func (m *aggregateCorr) Eval(ctx context.Context) (interface{}, error) { + corr := float64((float64(m.n)*m.sum_XY - m.sum_X*m.sum_Y)) / (math.Sqrt(float64((float64(m.n)*m.squareSum_X - m.sum_X*m.sum_X) * (float64(m.n)*m.squareSum_Y - m.sum_Y*m.sum_Y)))) + + d, err := pql.FromFloat64WithScale(corr, 6) + if err != nil { + return nil, err + } + return d, nil +} + +// corrPlanExpression handles CORR() - implement correlation coefficient +type corrPlanExpression struct { + arg1 types.PlanExpression + arg2 types.PlanExpression + returnDataType parser.ExprDataType +} + +var _ types.Aggregable = (*corrPlanExpression)(nil) + +func newCorrPlanExpression(arg1 types.PlanExpression, arg2 types.PlanExpression, returnDataType parser.ExprDataType) *corrPlanExpression { + return &corrPlanExpression{ + arg1: arg1, + arg2: arg2, + returnDataType: returnDataType, + } +} + +func (n *corrPlanExpression) Evaluate(currentRow []interface{}) (interface{}, error) { + return nil, sql3.NewErrInternalf("this should never be called") +} + +func (n *corrPlanExpression) NewBuffer() (types.AggregationBuffer, error) { + return NewAggCorrBuffer(n), nil +} + +func (n *corrPlanExpression) FirstChildExpr() types.PlanExpression { + return n.arg1 +} + +func (n *corrPlanExpression) Type() parser.ExprDataType { + return n.returnDataType +} + +func (n *corrPlanExpression) String() string { + return fmt.Sprintf("corr(%s, %s)", n.arg1.String(), n.arg2.String()) +} + +func (n *corrPlanExpression) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_expr"] = fmt.Sprintf("%T", n) + result["description"] = n.String() + result["dataType"] = n.Type().TypeDescription() + result["arg1"] = n.arg1.Plan() + result["arg2"] = n.arg2.Plan() + return result +} + +func (n *corrPlanExpression) Children() []types.PlanExpression { + return []types.PlanExpression{ + n.arg1, + n.arg2, + } +} + +func (n *corrPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) { + if len(children) != 2 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return newCorrPlanExpression(children[0], children[1], n.returnDataType), nil +} + +// aggregator for VAR() +type aggregateVar struct { + expr *varPlanExpression + + // to calculate mean + n int64 + sum float64 + + // we need to hang on to the values + // TODO(pok) - will need to spill these to disk for big result sets + values []float64 +} + +func NewAggVarBuffer(child *varPlanExpression) *aggregateVar { + return &aggregateVar{ + expr: child, + values: make([]float64, 0), + } +} + +func (m *aggregateVar) Update(ctx context.Context, row types.Row) error { + v, err := m.expr.arg.Evaluate(row) + if err != nil { + return err + } + + // skip if nil + if v == nil { + return nil + } + + var val float64 + + switch dataType := m.expr.arg.Type().(type) { + case *parser.DataTypeDecimal: + thisVal, ok := v.(pql.Decimal) + if !ok { + return sql3.NewErrInternalf("unexpected type conversion '%T'", v) + } + + val = thisVal.Float64() + + case *parser.DataTypeID: + thisVal, ok := v.(int64) + if !ok { + return sql3.NewErrInternalf("unexpected type conversion '%T'", v) + } + + val = float64(thisVal) + + case *parser.DataTypeInt: + thisVal, ok := v.(int64) + if !ok { + return sql3.NewErrInternalf("unexpected type conversion '%T'", v) + } + + val = float64(thisVal) + + default: + return sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", dataType) + } + + m.sum += val + m.n += 1 + m.values = append(m.values, val) + + return nil +} + +func (m *aggregateVar) Eval(ctx context.Context) (interface{}, error) { + + mean := m.sum / float64(m.n) + + var variance float64 + for _, v := range m.values { + variance += (v - mean) * (v - mean) + } + + variance = variance / float64(m.n) + + d, err := pql.FromFloat64WithScale(variance, 6) + if err != nil { + return nil, err + } + return d, nil +} + +// varPlanExpression handles VAR() - variance +type varPlanExpression struct { + arg types.PlanExpression + returnDataType parser.ExprDataType +} + +var _ types.Aggregable = (*varPlanExpression)(nil) + +func newVarPlanExpression(arg types.PlanExpression, returnDataType parser.ExprDataType) *varPlanExpression { + return &varPlanExpression{ + arg: arg, + returnDataType: returnDataType, + } +} + +func (n *varPlanExpression) Evaluate(currentRow []interface{}) (interface{}, error) { + return nil, sql3.NewErrInternalf("this should never be called") +} + +func (n *varPlanExpression) NewBuffer() (types.AggregationBuffer, error) { + return NewAggVarBuffer(n), nil +} + +func (n *varPlanExpression) FirstChildExpr() types.PlanExpression { + return n.arg +} + +func (n *varPlanExpression) Type() parser.ExprDataType { + return n.returnDataType +} + +func (n *varPlanExpression) String() string { + return fmt.Sprintf("var(%s)", n.arg.String()) +} + +func (n *varPlanExpression) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_expr"] = fmt.Sprintf("%T", n) + result["description"] = n.String() + result["dataType"] = n.Type().TypeDescription() + result["arg"] = n.arg.Plan() + return result +} + +func (n *varPlanExpression) Children() []types.PlanExpression { + return []types.PlanExpression{ + n.arg, + } +} + +func (n *varPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) { + if len(children) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return newVarPlanExpression(children[0], n.returnDataType), nil +} + // aggregator for LAST() type aggregateLast struct { val interface{} diff --git a/sql3/planner/expressionanalyzercall.go b/sql3/planner/expressionanalyzercall.go index 597fbd1ba..b75c2c8fa 100644 --- a/sql3/planner/expressionanalyzercall.go +++ b/sql3/planner/expressionanalyzercall.go @@ -124,6 +124,71 @@ func (p *ExecutionPlanner) analyzeCallExpression(ctx context.Context, call *pars //return the data type of the referenced column call.ResultDataType = ref.DataType() + case "CORR": + // can't do this on a * + if call.Star.IsValid() && len(call.Args) == 0 { + return nil, sql3.NewErrExpectedColumnReference(call.Star.Line, call.Star.Column) + } + + if len(call.Args) != 2 { + return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 2, len(call.Args)) + } + + // if it is a ref, we shouldn't do a corr on the _id + arg1 := call.Args[0] + + ref, ok := arg1.(*parser.QualifiedRef) + if ok && strings.EqualFold(ref.Column.Name, string(dax.PrimaryKeyFieldName)) { + return nil, sql3.NewErrIdColumnNotValidForAggregateFunction(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Name.Name) + } + + // make sure the ref is the right type + if !(typeIsInteger(arg1.DataType()) || typeIsDecimal(arg1.DataType()) || typeIsTimestamp(arg1.DataType())) { + return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(arg1.Pos().Line, arg1.Pos().Column) + } + + // if it is a ref, we shouldn't do a corr on the _id + arg2 := call.Args[1] + + ref, ok = arg2.(*parser.QualifiedRef) + if ok && strings.EqualFold(ref.Column.Name, string(dax.PrimaryKeyFieldName)) { + return nil, sql3.NewErrIdColumnNotValidForAggregateFunction(call.Args[1].Pos().Line, call.Args[1].Pos().Column, call.Name.Name) + } + + // make sure the ref is the right type + if !(typeIsInteger(arg2.DataType()) || typeIsDecimal(arg2.DataType()) || typeIsTimestamp(arg2.DataType())) { + return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(arg2.Pos().Line, arg2.Pos().Column) + } + + // return the data type of the referenced column + call.ResultDataType = parser.NewDataTypeDecimal(6) + + case "VAR": + // can't do this on a * + if call.Star.IsValid() && len(call.Args) == 0 { + return nil, sql3.NewErrExpectedColumnReference(call.Star.Line, call.Star.Column) + } + + if len(call.Args) != 1 { + return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 1, len(call.Args)) + } + + // first arg should be a qualified ref + arg1 := call.Args[0] + + ref, ok := arg1.(*parser.QualifiedRef) + if ok && strings.EqualFold(ref.Column.Name, string(dax.PrimaryKeyFieldName)) { + return nil, sql3.NewErrIdColumnNotValidForAggregateFunction(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Name.Name) + } + + // make sure the ref is the right type + if !(typeIsInteger(arg1.DataType()) || typeIsDecimal(arg1.DataType()) || typeIsTimestamp(arg1.DataType())) { + return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(arg1.Pos().Line, arg1.Pos().Column) + } + + // return the data type of the referenced column + call.ResultDataType = parser.NewDataTypeDecimal(6) + case "MIN", "MAX": // can't do an min/max on a * if call.Star.IsValid() && len(call.Args) == 0 { @@ -270,6 +335,15 @@ func (p *ExecutionPlanner) analyzeCallExpression(ctx context.Context, call *pars case "DATETIMEDIFF": return p.analyzeFunctionDateTimeDiff(call, scope) default: + // could be a udf - try to look it up in functions + fn, err := p.getFunctionByName(call.Name.Name) + if err != nil { + return nil, err + } + if fn != nil { + return p.analyzeUserDefinedFunction(call, scope, fn) + } + return nil, sql3.NewErrCallUnknownFunction(call.Name.NamePos.Line, call.Name.NamePos.Column, call.Name.Name) } return call, nil diff --git a/sql3/planner/expressionpql.go b/sql3/planner/expressionpql.go index c6f2022b7..48526a841 100644 --- a/sql3/planner/expressionpql.go +++ b/sql3/planner/expressionpql.go @@ -604,7 +604,7 @@ func planExprToValue(expr types.PlanExpression) (interface{}, error) { return expr.value, nil case *stringLiteralPlanExpression: return expr.value, nil - case *dateLiteralPlanExpression: + case *timestampLiteralPlanExpression: return expr.value, nil case *boolLiteralPlanExpression: return expr.value, nil diff --git a/sql3/planner/opbulkinsert.go b/sql3/planner/opbulkinsert.go index a8d309cb8..2586ad34b 100644 --- a/sql3/planner/opbulkinsert.go +++ b/sql3/planner/opbulkinsert.go @@ -867,7 +867,7 @@ func processColumnValue(rawValue interface{}, targetType parser.ExprDataType) (t if !ok { return nil, sql3.NewErrInternalf("unable to convert '%s", rawValue) } - return newDateLiteralPlanExpression(tval), nil + return newTimestampLiteralPlanExpression(tval), nil case *parser.DataTypeString: sval, ok := rawValue.(string) diff --git a/sql3/planner/opcopy.go b/sql3/planner/opcopy.go new file mode 100644 index 000000000..2ede81ff0 --- /dev/null +++ b/sql3/planner/opcopy.go @@ -0,0 +1,515 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + pilosa "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/pql" + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +// PlanOpCopy is a copy operator +type PlanOpCopy struct { + planner *ExecutionPlanner + targetTable string + url string + apiKey string + ddl string + ChildOp types.PlanOperator + + warnings []string +} + +func NewPlanOpCopy(planner *ExecutionPlanner, targetName string, url string, apiKey string, ddl string, child types.PlanOperator) *PlanOpCopy { + return &PlanOpCopy{ + planner: planner, + targetTable: targetName, + url: url, + apiKey: apiKey, + ddl: ddl, + ChildOp: child, + warnings: make([]string, 0), + } +} + +func (p *PlanOpCopy) Schema() types.Schema { + return types.Schema{} +} + +func (p *PlanOpCopy) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + child, err := p.ChildOp.Iterator(ctx, row) + if err != nil { + return nil, err + } + if p.url != "" { + return newRemoteCopyIterator(p.planner, p.targetTable, p.url, p.apiKey, p.ddl, p.ChildOp.Schema(), child), nil + } + return newCopyIterator(p.planner, p.targetTable, p.ddl, p.ChildOp.Schema(), child), nil +} + +func (p *PlanOpCopy) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + if len(children) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return NewPlanOpCopy(p.planner, p.targetTable, p.url, p.apiKey, p.ddl, children[0]), nil +} + +func (p *PlanOpCopy) Children() []types.PlanOperator { + return []types.PlanOperator{ + p.ChildOp, + } +} + +func (p *PlanOpCopy) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + result["_schema"] = p.Schema().Plan() + result["child"] = p.ChildOp.Plan() + result["child"] = p.ChildOp.Plan() + return result +} + +func (p *PlanOpCopy) String() string { + return "" +} + +func (p *PlanOpCopy) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpCopy) Warnings() []string { + return p.warnings +} + +func (p *PlanOpCopy) Expressions() []types.PlanExpression { + return []types.PlanExpression{} +} + +func (p *PlanOpCopy) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) { + if len(exprs) > 0 { + return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs)) + } + return p, nil +} + +type copyIterator struct { + planner *ExecutionPlanner + targetTableName string + copySchema types.Schema + ddl string + child types.RowIterator + hasStarted *struct{} +} + +func newCopyIterator(planner *ExecutionPlanner, targetTableName string, ddl string, copySchema types.Schema, childIter types.RowIterator) *copyIterator { + return ©Iterator{ + planner: planner, + targetTableName: targetTableName, + ddl: ddl, + copySchema: copySchema, + child: childIter, + } +} + +func (i *copyIterator) Next(ctx context.Context) (types.Row, error) { + if i.hasStarted == nil { + // parse and execute the ddl to create the table + ast, err := parser.NewParser(strings.NewReader(i.ddl)).ParseStatement() + if err != nil { + return nil, err + } + ct, ok := ast.(*parser.CreateTableStatement) + if !ok { + return nil, sql3.NewErrInternalf("unexpected ast type") + } + // analyze + err = i.planner.analyzeCreateTableStatement(ct) + if err != nil { + return nil, err + } + ctOp, err := i.planner.compileCreateTableStatement(ctx, ct) + if err != nil { + return nil, err + } + ctIter, err := ctOp.Iterator(context.Background(), nil) + if err != nil { + return nil, err + } + _, err = ctIter.Next(ctx) + if err != nil && err != types.ErrNoMoreRows { + return nil, err + } + + targetColumns := make([]*qualifiedRefPlanExpression, 0) + + for _, s := range i.copySchema { + targetColumns = append(targetColumns, newQualifiedRefPlanExpression(i.targetTableName, s.ColumnName, 0, s.Type)) + } + + // build an insert iterator for the target table + insertIter := &insertRowIter{ + planner: i.planner, + tableName: i.targetTableName, + targetColumns: targetColumns, + } + + batchCount := 0 + insertBatch := make([][]types.PlanExpression, 0) + + for { + // get a source row + row, err := i.child.Next(ctx) + if err != nil { + if err == types.ErrNoMoreRows { + break + } + return nil, err + } + + // add it to target batch + + irow := make([]types.PlanExpression, len(row)) + for i, s := range i.copySchema { + switch ty := s.Type.(type) { + case *parser.DataTypeID, *parser.DataTypeInt: + val, ok := row[i].(int64) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + irow[i] = newIntLiteralPlanExpression(val) + + case *parser.DataTypeDecimal: + val, ok := row[i].(pql.Decimal) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + irow[i] = newFloatLiteralPlanExpression(val.String()) + + case *parser.DataTypeString: + val, ok := row[i].(string) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + irow[i] = newStringLiteralPlanExpression(val) + + case *parser.DataTypeBool: + val, ok := row[i].(bool) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + irow[i] = newBoolLiteralPlanExpression(val) + + case *parser.DataTypeTimestamp: + val, ok := row[i].(time.Time) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + irow[i] = newTimestampLiteralPlanExpression(val) + + case *parser.DataTypeStringSet: + val, ok := row[i].([]string) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + + members := make([]types.PlanExpression, 0) + for _, m := range val { + members = append(members, newStringLiteralPlanExpression(m)) + } + irow[i] = newExprSetLiteralPlanExpression(members, parser.NewDataTypeStringSet()) + + case *parser.DataTypeIDSet: + val, ok := row[i].([]int64) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + + members := make([]types.PlanExpression, 0) + for _, m := range val { + members = append(members, newIntLiteralPlanExpression(m)) + } + irow[i] = newExprSetLiteralPlanExpression(members, parser.NewDataTypeIDSet()) + + default: + return nil, sql3.NewErrInternalf("unhandled type '%T'", ty) + } + } + insertBatch = append(insertBatch, irow) + + // inc batch count + batchCount += 1 + if batchCount > 1000 { + // do the insert + insertIter.insertValues = insertBatch + _, err = insertIter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return nil, err + } + // reset + batchCount = 0 + insertBatch = make([][]types.PlanExpression, 0) + } + } + if len(insertBatch) > 0 { + // do the insert + insertIter.insertValues = insertBatch + _, err = insertIter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return nil, err + } + } + + i.hasStarted = &struct{}{} + } + return nil, types.ErrNoMoreRows +} + +type remoteCopyIterator struct { + planner *ExecutionPlanner + targetTableName string + copySchema types.Schema + url string + apiKey string + ddl string + child types.RowIterator + hasStarted *struct{} +} + +func newRemoteCopyIterator(planner *ExecutionPlanner, targetTableName string, url string, apiKey string, ddl string, copySchema types.Schema, childIter types.RowIterator) *remoteCopyIterator { + return &remoteCopyIterator{ + planner: planner, + targetTableName: targetTableName, + url: url, + apiKey: apiKey, + ddl: ddl, + copySchema: copySchema, + child: childIter, + } +} + +func (i *remoteCopyIterator) remoteExec(ctx context.Context, sql string) (*pilosa.WireQueryResponse, error) { + // Create HTTP request. + req, err := http.NewRequest("POST", i.url, strings.NewReader(sql)) + if err != nil { + return nil, sql3.NewErrInternalf("error executing remotely: %s", err.Error()) + } + + req.Header.Set("Content-Length", strconv.Itoa(len(sql))) + req.Header.Set("Content-Type", "text/plain") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+i.planner.systemAPI.Version()) + if len(i.apiKey) > 0 { + req.Header.Set("X-API-Key", i.apiKey) + } + + // Execute request against the host. + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, sql3.NewErrInternalf("error executing remotely: %s", err.Error()) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, sql3.NewErrInternalf("error executing remotely: %s", err.Error()) + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + if resp.StatusCode == 401 { + return nil, sql3.NewErrRemoteUnauthorized(0, 0, i.url) + } + // we have an error + return nil, sql3.NewErrInternalf("error executing remotely: %d, %s", resp.StatusCode, string(body)) + } + + sqlResponse := &pilosa.WireQueryResponse{} + err = sqlResponse.UnmarshalJSONTyped([]byte(body), true) + if err != nil { + return nil, sql3.NewErrInternalf("error executing remotely: %s", err.Error()) + } + + if len(sqlResponse.Error) > 0 { + return nil, sql3.NewErrInternalf("error executing remotely: %s", sqlResponse.Error) + } + + return sqlResponse, nil +} + +func (i *remoteCopyIterator) Next(ctx context.Context) (types.Row, error) { + if i.hasStarted == nil { + // execute the ddl to create the table + _, err := i.remoteExec(ctx, i.ddl) + if err != nil { + return nil, err + } + + // build bulk insert statement + var buf bytes.Buffer + buf.WriteString("bulk insert into ") + fmt.Fprintf(&buf, "%s", i.targetTableName) + buf.WriteString(" (") + + for i, s := range i.copySchema { + if i > 0 { + buf.WriteString(", ") + } + fmt.Fprintf(&buf, "%s", s.ColumnName) + } + buf.WriteString(") map (") + for i, s := range i.copySchema { + if i > 0 { + buf.WriteString(", ") + } + fmt.Fprintf(&buf, "'$._%d' %s", i, s.Type.TypeDescription()) + } + buf.WriteString(") from x'") + header := buf.String() + + batchCount := 0 + var batchBuf bytes.Buffer + + for { + // get a source row + row, err := i.child.Next(ctx) + if err != nil { + if err == types.ErrNoMoreRows { + break + } + return nil, err + } + + // add it to target batch + var rowBuf bytes.Buffer + rowBuf.WriteString("{") + for i, s := range i.copySchema { + if i > 0 { + rowBuf.WriteString(",") + } + fmt.Fprintf(&rowBuf, `"_%d":`, i) + + if row[i] == nil { + rowBuf.WriteString("null") + continue + } + + switch ty := s.Type.(type) { + case *parser.DataTypeID, *parser.DataTypeInt: + val, ok := row[i].(int64) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + fmt.Fprintf(&rowBuf, "%d", val) + + case *parser.DataTypeString: + val, ok := row[i].(string) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + // escape single quotes + val = strings.ReplaceAll(val, `'`, `''`) + // and double quotes + val = strings.ReplaceAll(val, `"`, `\"`) + // and line feeds + if strings.Contains(val, "\n") { + val = strings.ReplaceAll(val, "\n", "\\n") + } + fmt.Fprintf(&rowBuf, `"%s"`, val) + + case *parser.DataTypeBool: + val, ok := row[i].(bool) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + fmt.Fprintf(&rowBuf, "%v", val) + + case *parser.DataTypeTimestamp: + val, ok := row[i].(time.Time) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + fmt.Fprintf(&rowBuf, `"%s"`, val.Format(time.RFC3339Nano)) + + case *parser.DataTypeStringSet: + val, ok := row[i].([]string) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + rowBuf.WriteString("[") + for j, s := range val { + if j > 0 { + rowBuf.WriteString(",") + } + fmt.Fprintf(&rowBuf, `"%s"`, s) + } + rowBuf.WriteString("]") + + case *parser.DataTypeIDSet: + val, ok := row[i].([]int64) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i]) + } + rowBuf.WriteString("[") + for j, s := range val { + if j > 0 { + rowBuf.WriteString(",") + } + fmt.Fprintf(&rowBuf, `%d`, s) + } + rowBuf.WriteString("]") + + default: + return nil, sql3.NewErrInternalf("unhandled type '%T'", ty) + } + } + rowBuf.WriteString("}\n") + batchBuf.Write(rowBuf.Bytes()) + + // inc batch count + batchCount += 1 + if batchCount > 10000 { + // do the insert + + var reqBuf bytes.Buffer + reqBuf.WriteString(header) + reqBuf.Write(batchBuf.Bytes()) + reqBuf.WriteString("' with batchsize 10000 input 'STREAM' format 'NDJSON'") + + _, err := i.remoteExec(ctx, reqBuf.String()) + if err != nil { + return nil, err + } + + // reset + batchCount = 0 + batchBuf.Reset() + } + } + if batchCount > 0 { + // do the insert + + var reqBuf bytes.Buffer + reqBuf.WriteString(header) + reqBuf.Write(batchBuf.Bytes()) + reqBuf.WriteString("' with batchsize 10000 input 'STREAM' format 'NDJSON'") + + _, err := i.remoteExec(ctx, reqBuf.String()) + if err != nil { + return nil, err + } + } + + i.hasStarted = &struct{}{} + } + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/opcreatefunction.go b/sql3/planner/opcreatefunction.go new file mode 100644 index 000000000..0c84c88aa --- /dev/null +++ b/sql3/planner/opcreatefunction.go @@ -0,0 +1,104 @@ +// Copyright 2023 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "fmt" + + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +// PlanOpCreateFunction implements the CREATE FUNCTION operator +type PlanOpCreateFunction struct { + planner *ExecutionPlanner + function *functionSystemObject + ifNotExists bool + warnings []string +} + +func NewPlanOpCreateFunction(planner *ExecutionPlanner, ifNotExists bool, function *functionSystemObject) *PlanOpCreateFunction { + return &PlanOpCreateFunction{ + planner: planner, + function: function, + ifNotExists: ifNotExists, + warnings: make([]string, 0), + } +} + +func (p *PlanOpCreateFunction) Schema() types.Schema { + return types.Schema{} +} + +func (p *PlanOpCreateFunction) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + return newCreateFunctionIter(p.planner, p.ifNotExists, p.function), nil +} + +func (p *PlanOpCreateFunction) Children() []types.PlanOperator { + return []types.PlanOperator{} +} + +func (p *PlanOpCreateFunction) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + if len(children) != 0 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return NewPlanOpCreateFunction(p.planner, p.ifNotExists, p.function), nil +} + +func (p *PlanOpCreateFunction) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + result["_schema"] = p.Schema().Plan() + result["model"] = p.function.name + return result +} + +func (p *PlanOpCreateFunction) String() string { + return "" +} + +func (p *PlanOpCreateFunction) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpCreateFunction) Warnings() []string { + var w []string + w = append(w, p.warnings...) + return w +} + +type createFunctionIter struct { + planner *ExecutionPlanner + function *functionSystemObject + ifNotExists bool +} + +func newCreateFunctionIter(planner *ExecutionPlanner, ifNotExists bool, function *functionSystemObject) *createFunctionIter { + return &createFunctionIter{ + planner: planner, + function: function, + ifNotExists: ifNotExists, + } +} + +func (i *createFunctionIter) Next(ctx context.Context) (types.Row, error) { + // now check in the functions table to see if it is exists + v, err := i.planner.getFunctionByName(i.function.name) + if err != nil { + return nil, err + } + if v != nil { + if i.ifNotExists { + return nil, types.ErrNoMoreRows + } + return nil, sql3.NewErrViewExists(0, 0, i.function.name) + } + + // now store the view into fb_functions + err = i.planner.insertFunction(i.function) + if err != nil { + return nil, err + } + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/opcreatemodel.go b/sql3/planner/opcreatemodel.go new file mode 100644 index 000000000..39136e0ca --- /dev/null +++ b/sql3/planner/opcreatemodel.go @@ -0,0 +1,279 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/featurebasedb/featurebase/v3/pql" + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + uuid "github.com/satori/go.uuid" +) + +// PlanOpCreateModel implements the CREATE MODEL operator +type PlanOpCreateModel struct { + ChildOp types.PlanOperator + planner *ExecutionPlanner + model *modelSystemObject + warnings []string +} + +func NewPlanOpCreateModel(planner *ExecutionPlanner, model *modelSystemObject, child types.PlanOperator) *PlanOpCreateModel { + return &PlanOpCreateModel{ + ChildOp: child, + planner: planner, + model: model, + warnings: make([]string, 0), + } +} + +func (p *PlanOpCreateModel) Schema() types.Schema { + return types.Schema{} +} + +func (p *PlanOpCreateModel) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + // get the query iterator + iter, err := p.ChildOp.Iterator(ctx, row) + if err != nil { + return nil, err + } + + switch strings.ToLower(p.model.modelType) { + case "linear_regresssion": + return newCreateModelIter(p.planner, p.model, newLinearRegressionModelIter(p.planner, p.model, p.ChildOp.Schema(), iter)), nil + + default: + return nil, sql3.NewErrInternalf("unexpected model tyoe '%s'", p.model.modelType) + } +} + +func (p *PlanOpCreateModel) Children() []types.PlanOperator { + return []types.PlanOperator{ + p.ChildOp, + } +} + +func (p *PlanOpCreateModel) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + if len(children) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return NewPlanOpCreateModel(p.planner, p.model, children[0]), nil +} + +func (p *PlanOpCreateModel) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + sc := make([]string, 0) + for _, e := range p.Schema() { + sc = append(sc, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeDescription())) + } + result["_schema"] = sc + result["model"] = p.model.name // TODO(pok) - add a Plan() method here (or some such) + result["child"] = p.ChildOp.Plan() + return result +} + +func (p *PlanOpCreateModel) String() string { + return "" +} + +func (p *PlanOpCreateModel) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpCreateModel) Warnings() []string { + var w []string + w = append(w, p.warnings...) + w = append(w, p.ChildOp.Warnings()...) + return w +} + +type createModelIter struct { + child types.RowIterator + planner *ExecutionPlanner + model *modelSystemObject + hasStarted *struct{} +} + +func newCreateModelIter(planner *ExecutionPlanner, model *modelSystemObject, child types.RowIterator) *createModelIter { + return &createModelIter{ + planner: planner, + model: model, + child: child, + } +} + +func (i *createModelIter) Next(ctx context.Context) (types.Row, error) { + if i.hasStarted == nil { + // store the model into fb_models and set the model status to 'training' + i.model.status = "TRAINING" + err := i.planner.insertModel(i.model) + if err != nil { + return nil, err + } + + // do the actual training + _, err = i.child.Next(ctx) + if err != nil && err != types.ErrNoMoreRows { + return nil, err + } + + // update the model to ready + i.model.status = "READY" + err = i.planner.updateModel(i.model) + if err != nil { + return nil, err + } + i.hasStarted = &struct{}{} + } + return nil, types.ErrNoMoreRows +} + +type linearRegressionModelIter struct { + child types.RowIterator + planner *ExecutionPlanner + model *modelSystemObject + childSchema types.Schema + hasStarted *struct{} +} + +func newLinearRegressionModelIter(planner *ExecutionPlanner, model *modelSystemObject, childSchema types.Schema, child types.RowIterator) *linearRegressionModelIter { + return &linearRegressionModelIter{ + planner: planner, + model: model, + childSchema: childSchema, + child: child, + } +} + +func (i *linearRegressionModelIter) Next(ctx context.Context) (types.Row, error) { + if i.hasStarted == nil { + // this is linear regression now, so we actually 'train' when we predict (later) + // for now we just store the values from the query in fb_model_data + + // delete anything from fb_model_data for this model + err := i.planner.ensureModelDataSystemTableExists() + if err != nil { + return nil, err + } + diter := &filteredDeleteRowIter{ + planner: i.planner, + tableName: "fb_model_data", + filter: newBinOpPlanExpression( + newQualifiedRefPlanExpression("fb_model_data", "model_id", 0, parser.NewDataTypeString()), + parser.EQ, + newStringLiteralPlanExpression(i.model.name), + parser.NewDataTypeBool(), + ), + } + _, err = diter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return nil, err + } + + iter := &insertRowIter{ + planner: i.planner, + tableName: "fb_model_data", + targetColumns: []*qualifiedRefPlanExpression{ + newQualifiedRefPlanExpression("fb_model_data", "_id", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_model_data", "model_id", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_model_data", "data", 0, parser.NewDataTypeString()), + }, + insertValues: [][]types.PlanExpression{}, + } + + trainingRefs := make([]*qualifiedRefPlanExpression, 0) + + // make sure label column exists and is type compatible with float + labelColumn := i.model.labels[0] + found := false + for i, s := range i.childSchema { + if strings.EqualFold(labelColumn, s.ColumnName) { + if !typesAreAssignmentCompatible(parser.NewDataTypeDecimal(4), s.Type) { + return nil, sql3.NewErrInternalf("types not assignment compatible") + } + trainingRefs = append(trainingRefs, newQualifiedRefPlanExpression("", labelColumn, i, s.Type)) + found = true + break + } + } + if !found { + return nil, sql3.NewErrInternalf("label column found found") + } + + // make sure input columns exists and are type compatible with float + for _, ic := range i.model.inputColumns { + found := false + for i, s := range i.childSchema { + if strings.EqualFold(ic, s.ColumnName) { + if !typesAreAssignmentCompatible(parser.NewDataTypeDecimal(4), s.Type) { + return nil, sql3.NewErrInternalf("types not assignment compatible") + } + trainingRefs = append(trainingRefs, newQualifiedRefPlanExpression("", ic, i, s.Type)) + found = true + break + } + } + if !found { + return nil, sql3.NewErrInternalf("input column found found") + } + } + + // go run the query and iterate + for { + row, err := i.child.Next(ctx) + if err != nil { + if err == types.ErrNoMoreRows { + break + } + return nil, err + } + + fdata := make([]float64, 0) + + for _, ref := range trainingRefs { + val, err := ref.Evaluate(row) + if err != nil { + return nil, err + } + cval, err := coerceValue(ref.dataType, parser.NewDataTypeDecimal(4), val, parser.Pos{Line: 0, Column: 0}) + if err != nil { + return nil, err + } + dval := cval.(pql.Decimal) + fdata = append(fdata, dval.Float64()) + } + + data, err := json.Marshal(fdata) + if err != nil { + return nil, err + } + + rowID, err := uuid.NewV4() + if err != nil { + return nil, err + } + tuple := []types.PlanExpression{ + newStringLiteralPlanExpression(rowID.String()), + newStringLiteralPlanExpression(i.model.name), + newStringLiteralPlanExpression(string(data)), + } + iter.insertValues = append(iter.insertValues, tuple) + + fmt.Printf("%v", row) + + } + _, err = iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return nil, err + } + + i.hasStarted = &struct{}{} + } + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/opdropmodel.go b/sql3/planner/opdropmodel.go new file mode 100644 index 000000000..86cc3ee7b --- /dev/null +++ b/sql3/planner/opdropmodel.go @@ -0,0 +1,102 @@ +// Copyright 2023 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "fmt" + + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +// PlanOpDropModel plan operator to drop a view. +type PlanOpDropModel struct { + planner *ExecutionPlanner + modelName string + ifExists bool + warnings []string +} + +func NewPlanOpDropModel(p *ExecutionPlanner, ifExists bool, modelName string) *PlanOpDropModel { + return &PlanOpDropModel{ + planner: p, + modelName: modelName, + ifExists: ifExists, + warnings: make([]string, 0), + } +} + +func (p *PlanOpDropModel) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + result["modelName"] = p.modelName + result["isExists"] = p.ifExists + return result +} + +func (p *PlanOpDropModel) String() string { + return "" +} + +func (p *PlanOpDropModel) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpDropModel) Warnings() []string { + return p.warnings +} + +func (p *PlanOpDropModel) Schema() types.Schema { + return types.Schema{} +} + +func (p *PlanOpDropModel) Children() []types.PlanOperator { + return []types.PlanOperator{} +} + +func (p *PlanOpDropModel) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + return &dropModelRowIter{ + planner: p.planner, + ifExists: p.ifExists, + modelName: p.modelName, + }, nil +} + +func (p *PlanOpDropModel) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + return nil, nil +} + +type dropModelRowIter struct { + planner *ExecutionPlanner + ifExists bool + modelName string +} + +var _ types.RowIterator = (*dropModelRowIter)(nil) + +func (i *dropModelRowIter) Next(ctx context.Context) (types.Row, error) { + err := i.planner.checkAccess(ctx, i.modelName, accessTypeDropObject) + if err != nil { + return nil, err + } + + // check in the models table to see if it exists + v, err := i.planner.getModelByName(i.modelName) + if err != nil { + return nil, err + } + if v == nil { + if i.ifExists { + return nil, types.ErrNoMoreRows + } + return nil, sql3.NewErrModelNotFound(0, 0, i.modelName) + } + + err = i.planner.deleteModel(i.modelName) + if err != nil { + return nil, err + } + + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/oppredict.go b/sql3/planner/oppredict.go new file mode 100644 index 000000000..ac9e4f233 --- /dev/null +++ b/sql3/planner/oppredict.go @@ -0,0 +1,253 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/featurebasedb/featurebase/v3/pql" + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "github.com/sajari/regression" +) + +// PlanOpPredict is an operator for a PREDICT +type PlanOpPredict struct { + ChildOp types.PlanOperator + planner *ExecutionPlanner + model *modelSystemObject + warnings []string +} + +func NewPlanOpPredict(planner *ExecutionPlanner, model *modelSystemObject, child types.PlanOperator) *PlanOpPredict { + return &PlanOpPredict{ + ChildOp: child, + planner: planner, + model: model, + warnings: make([]string, 0), + } +} + +func (p *PlanOpPredict) Schema() types.Schema { + result := make(types.Schema, 0) + + switch strings.ToLower(p.model.modelType) { + case "linear_regresssion": + labelName := p.model.labels[0] + + result = append(result, &types.PlannerColumn{ + ColumnName: fmt.Sprintf("predicted_%s", labelName), + RelationName: "", + AliasName: "", + // we need to get this type from somewhere...probably needs to be stored in the model def + Type: &parser.DataTypeDecimal{ + Scale: 4, + }, + }) + + default: + // don't add anything + } + // add the columns from the select + result = append(result, p.ChildOp.Schema()...) + + return result +} + +func (p *PlanOpPredict) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + // get the query iterator + iter, err := p.ChildOp.Iterator(ctx, row) + if err != nil { + return nil, err + } + + switch strings.ToLower(p.model.modelType) { + case "linear_regresssion": + return newLinearRegressionPredictIter(p.planner, p.model, p.ChildOp.Schema(), iter), nil + + default: + return nil, sql3.NewErrInternalf("unexpected model tyoe '%s'", p.model.modelType) + } +} + +func (p *PlanOpPredict) Children() []types.PlanOperator { + return []types.PlanOperator{ + p.ChildOp, + } +} + +func (p *PlanOpPredict) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + if len(children) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return NewPlanOpPredict(p.planner, p.model, children[0]), nil +} + +func (p *PlanOpPredict) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + sc := make([]string, 0) + for _, e := range p.Schema() { + sc = append(sc, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeDescription())) + } + result["_schema"] = sc + result["model"] = p.model.name + result["child"] = p.ChildOp.Plan() + return result +} + +func (p *PlanOpPredict) String() string { + return "" +} + +func (p *PlanOpPredict) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpPredict) Warnings() []string { + var w []string + w = append(w, p.warnings...) + if p.ChildOp != nil { + w = append(w, p.ChildOp.Warnings()...) + } + return w +} + +type linearRegressionPredictIter struct { + child types.RowIterator + planner *ExecutionPlanner + model *modelSystemObject + regres *regression.Regression + childSchema types.Schema + inferenceRefs []*qualifiedRefPlanExpression + hasStarted *struct{} +} + +func newLinearRegressionPredictIter(planner *ExecutionPlanner, model *modelSystemObject, childSchema types.Schema, child types.RowIterator) *linearRegressionPredictIter { + return &linearRegressionPredictIter{ + planner: planner, + model: model, + child: child, + childSchema: childSchema, + regres: new(regression.Regression), + inferenceRefs: make([]*qualifiedRefPlanExpression, 0), + } +} + +func (i *linearRegressionPredictIter) Next(ctx context.Context) (types.Row, error) { + if i.hasStarted == nil { + // label column + i.regres.SetObserved("Murders per annum per 1,000,000 inhabitants") + // input columns + i.regres.SetVar(0, "Inhabitants") + i.regres.SetVar(1, "Percent with incomes below $5000") + i.regres.SetVar(2, "Percent unemployed") + + // go get the 'training set' from fb_model_data + iter := &tableScanRowIter{ + planner: i.planner, + tableName: "fb_model_data", + columns: []string{ + "_id", + "model_id", + "data", + }, + predicate: newBinOpPlanExpression( + newQualifiedRefPlanExpression("fb_model_data", "model_id", 0, parser.NewDataTypeString()), + parser.EQ, + newStringLiteralPlanExpression(i.model.name), + parser.NewDataTypeBool(), + ), + } + + for { + row, err := iter.Next(context.Background()) + if err != nil { + if err == types.ErrNoMoreRows { + break + } + return nil, err + } + fdata := make([]float64, 0) + err = json.Unmarshal([]byte(row[2].(string)), &fdata) + if err != nil { + return nil, err + } + label := fdata[0] + vars := fdata[1:] + i.regres.Train(regression.DataPoint(label, vars)) + } + + // run the regression + err := i.regres.Run() + if err != nil { + return nil, err + } + + fmt.Printf("Regression formula:\n%v\n", i.regres.Formula) + fmt.Printf("Regression:\n%s\n", i.regres) + + for _, ic := range i.model.inputColumns { + found := false + for j, s := range i.childSchema { + if strings.EqualFold(ic, s.ColumnName) { + if !typesAreAssignmentCompatible(parser.NewDataTypeDecimal(4), s.Type) { + return nil, sql3.NewErrInternalf("types not assignment compatible") + } + i.inferenceRefs = append(i.inferenceRefs, newQualifiedRefPlanExpression("", ic, j, s.Type)) + found = true + break + } + } + if !found { + return nil, sql3.NewErrInternalf("input column found found") + } + } + + i.hasStarted = &struct{}{} + } + + childrow, err := i.child.Next(ctx) + if err != nil { + return nil, err + } + + // construct the inference data + inferenceData := make([]float64, len(i.inferenceRefs)) + for j, ref := range i.inferenceRefs { + + val, err := ref.Evaluate(childrow) + if err != nil { + return nil, err + } + cval, err := coerceValue(ref.dataType, parser.NewDataTypeDecimal(4), val, parser.Pos{Line: 0, Column: 0}) + if err != nil { + return nil, err + } + dval := cval.(pql.Decimal) + inferenceData[j] = dval.Float64() + } + + // do the prediction + prediction, err := i.regres.Predict(inferenceData) + if err != nil { + return nil, err + } + + // turn the predition into a decimal + dprediction, err := pql.FromFloat64WithScale(prediction, 4) + if err != nil { + return nil, err + } + + // make an output row + row := make(types.Row, len(childrow)+1) + row[0] = dprediction + copy(row[1:], childrow) + + return row, nil +} diff --git a/sql3/planner/opsystemtable.go b/sql3/planner/opsystemtable.go index c6c2c74f4..ec21f5584 100644 --- a/sql3/planner/opsystemtable.go +++ b/sql3/planner/opsystemtable.go @@ -9,6 +9,7 @@ import ( "sort" pilosa "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/dax" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" @@ -500,6 +501,107 @@ type fbTableDDLRowIter struct { var _ types.RowIterator = (*fbTableDDLRowIter)(nil) +func generateTableDDL(tbl *dax.Table, newName string) string { + + var buf bytes.Buffer + buf.WriteString("create table ") + if len(newName) > 0 { + fmt.Fprintf(&buf, "%s", newName) + } else { + fmt.Fprintf(&buf, "%s", tbl.Name) + } + buf.WriteString(" (") + + for idx, col := range tbl.Fields { + if idx > 0 { + buf.WriteString(", ") + } + fmt.Fprintf(&buf, "%s", col.Name) + dataType := fieldSQLDataType(pilosa.FieldToFieldInfo(col)) + fmt.Fprintf(&buf, " %s", dataType.TypeDescription()) + + switch dt := dataType.(type) { + case *parser.DataTypeID, *parser.DataTypeString: + if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { + fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) + } + if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 { + // if we still have the default, we need to print that out if we have a non-default size + if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { + fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) + } + fmt.Fprintf(&buf, " size %d", col.Options.CacheSize) + } + + case *parser.DataTypeIDSet, *parser.DataTypeStringSet: + if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { + fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) + } + if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 { + // if we still have the default, we need to print that out if we have a non-default size + if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { + fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) + } + fmt.Fprintf(&buf, " size %d", col.Options.CacheSize) + } + + case *parser.DataTypeIDSetQuantum, *parser.DataTypeStringSetQuantum: + if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { + fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) + } + if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 { + // if we still have the default, we need to print that out if we have a non-default size + if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { + fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) + } + fmt.Fprintf(&buf, " size %d", col.Options.CacheSize) + } + if !col.Options.TimeQuantum.IsEmpty() { + fmt.Fprintf(&buf, " timequantum '%s'", col.Options.TimeQuantum) + } + if col.Options.TTL > 0 { + fmt.Fprintf(&buf, " ttl '%s'", col.Options.TTL.String()) + } + + case *parser.DataTypeInt: + minValue, maxValue := pql.MinMax(0) + + min := col.Options.Min + if !min.EqualTo(minValue) { + fmt.Fprintf(&buf, " min %d", min.ToInt64(0)) + } + + max := col.Options.Max + if !max.EqualTo(maxValue) { + fmt.Fprintf(&buf, " max %d", max.ToInt64(0)) + } + + case *parser.DataTypeDecimal: + minValue, maxValue := pql.MinMax(dt.Scale) + + min := col.Options.Min + if !min.EqualTo(minValue) { + fmt.Fprintf(&buf, " min %v", min) + } + + max := col.Options.Max + if !max.EqualTo(maxValue) { + fmt.Fprintf(&buf, " max %v", max) + } + + case *parser.DataTypeTimestamp: + if len(col.Options.TimeUnit) > 0 { + fmt.Fprintf(&buf, " timeunit '%s'", col.Options.TimeUnit) + } + // TODO(pok) how do we get epoch out of col? + + } + } + buf.WriteString(");") + + return buf.String() +} + func (i *fbTableDDLRowIter) Next(ctx context.Context) (types.Row, error) { if i.result == nil { tbls, err := i.planner.schemaAPI.Tables(ctx) @@ -512,98 +614,7 @@ func (i *fbTableDDLRowIter) Next(ctx context.Context) (types.Row, error) { for idx, tbl := range tbls { // build the ddl for this table - var buf bytes.Buffer - buf.WriteString("create table ") - fmt.Fprintf(&buf, "%s", tbl.Name) - buf.WriteString(" (") - - for idx, col := range tbl.Fields { - if idx > 0 { - buf.WriteString(", ") - } - fmt.Fprintf(&buf, "%s", col.Name) - dataType := fieldSQLDataType(pilosa.FieldToFieldInfo(col)) - fmt.Fprintf(&buf, " %s", dataType.TypeDescription()) - - switch dt := dataType.(type) { - case *parser.DataTypeID, *parser.DataTypeString: - if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { - fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) - } - if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 { - // if we still have the default, we need to print that out if we have a non-default size - if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { - fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) - } - fmt.Fprintf(&buf, " size %d", col.Options.CacheSize) - } - - case *parser.DataTypeIDSet, *parser.DataTypeStringSet: - if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { - fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) - } - if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 { - // if we still have the default, we need to print that out if we have a non-default size - if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { - fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) - } - fmt.Fprintf(&buf, " size %d", col.Options.CacheSize) - } - - case *parser.DataTypeIDSetQuantum, *parser.DataTypeStringSetQuantum: - if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { - fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) - } - if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 { - // if we still have the default, we need to print that out if we have a non-default size - if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 { - fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType) - } - fmt.Fprintf(&buf, " size %d", col.Options.CacheSize) - } - if !col.Options.TimeQuantum.IsEmpty() { - fmt.Fprintf(&buf, " timequantum '%s'", col.Options.TimeQuantum) - } - if col.Options.TTL > 0 { - fmt.Fprintf(&buf, " ttl '%s'", col.Options.TTL.String()) - } - - case *parser.DataTypeInt: - minValue, maxValue := pql.MinMax(0) - - min := col.Options.Min - if !min.EqualTo(minValue) { - fmt.Fprintf(&buf, " min %d", min.ToInt64(0)) - } - - max := col.Options.Max - if !max.EqualTo(maxValue) { - fmt.Fprintf(&buf, " max %d", max.ToInt64(0)) - } - - case *parser.DataTypeDecimal: - minValue, maxValue := pql.MinMax(dt.Scale) - - min := col.Options.Min - if !min.EqualTo(minValue) { - fmt.Fprintf(&buf, " min %v", min) - } - - max := col.Options.Max - if !max.EqualTo(maxValue) { - fmt.Fprintf(&buf, " max %v", max) - } - - case *parser.DataTypeTimestamp: - if len(col.Options.TimeUnit) > 0 { - fmt.Fprintf(&buf, " timeunit '%s'", col.Options.TimeUnit) - } - // TODO(pok) how do we get epoch out of col? - - } - } - buf.WriteString(");") - ddl := buf.String() + ddl := generateTableDDL(tbl, "") i.result[idx] = &fbTableDDLRow{ id: string(tbl.Name), diff --git a/sql3/planner/planoptimizer.go b/sql3/planner/planoptimizer.go index 6733b1510..ffae15cc4 100644 --- a/sql3/planner/planoptimizer.go +++ b/sql3/planner/planoptimizer.go @@ -1,4 +1,4 @@ -// Copyright 2021 Molecula Corp. All rights reserved. +// Copyright 2023 Molecula Corp. All rights reserved. package planner @@ -14,6 +14,8 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) +//TODO(pok) give every expression an id 'Expr1234' and use that to match on +//TODO(pok) have a rule to eliminate PlanOpRelAlias //TODO(pok) push filter down into join condition if terms reference either side of join //TODO(pok) push order by down as far as possible //TODO(pok) you can't group by _id in PQL, so we need to not use a PQL group by operator here @@ -707,6 +709,10 @@ func tryToReplaceGroupByWithPQLAggregate(ctx context.Context, a *ExecutionPlanne } thisNode.Aggregates[i] = newAgg + // these two can't be done in PQL + case *corrPlanExpression, *varPlanExpression: + return thisNode, true, nil + case types.Aggregable: switch ref := aggregable.FirstChildExpr().(type) { case *qualifiedRefPlanExpression: diff --git a/sql3/planner/systemobjects.go b/sql3/planner/systemobjects.go index 8fedda5bd..86a05393f 100644 --- a/sql3/planner/systemobjects.go +++ b/sql3/planner/systemobjects.go @@ -4,6 +4,7 @@ package planner import ( "context" + "encoding/json" "time" pilosa "github.com/featurebasedb/featurebase/v3" @@ -18,6 +19,20 @@ type viewSystemObject struct { statement string } +type functionSystemObject struct { + name string + language string + body string +} + +type modelSystemObject struct { + name string + status string + modelType string + labels []string + inputColumns []string +} + func (p *ExecutionPlanner) ensureViewsSystemTableExists(ctx context.Context) error { _, err := p.schemaAPI.TableByName(ctx, "fb_views") if err != nil { @@ -178,8 +193,8 @@ func (p *ExecutionPlanner) insertView(ctx context.Context, view *viewSystemObjec newStringLiteralPlanExpression(view.statement), newStringLiteralPlanExpression(""), newStringLiteralPlanExpression(""), - newDateLiteralPlanExpression(createTime), - newDateLiteralPlanExpression(createTime), + newTimestampLiteralPlanExpression(createTime), + newTimestampLiteralPlanExpression(createTime), }, }, } @@ -212,7 +227,7 @@ func (p *ExecutionPlanner) updateView(ctx context.Context, view *viewSystemObjec newStringLiteralPlanExpression(view.name), newStringLiteralPlanExpression(view.statement), newStringLiteralPlanExpression(""), - newDateLiteralPlanExpression(updateTime), + newTimestampLiteralPlanExpression(updateTime), }, }, } @@ -245,3 +260,621 @@ func (p *ExecutionPlanner) deleteView(ctx context.Context, viewName string) erro } return nil } + +func (p *ExecutionPlanner) ensureFunctionsSystemTableExists() error { + _, err := p.schemaAPI.TableByName(context.Background(), "fb_functions") + if err != nil { + if !isTableNotFoundError(err) { + return err + } + + // create table fb_functions ( + // _id string + // name string + // language string + // body string + // owner string + // updated_by string + // created_at timestamp + // updated_at timestamp + // ); + + // if it doesn't, create it by making the appropriate iterator + iter := &createTableRowIter{ + planner: p, + tableName: "fb_functions", + failIfExists: false, + isKeyed: true, + keyPartitions: 0, + columns: []*createTableField{ + { + planner: p, + name: "name", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "language", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "body", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "owner", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "updated_by", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "created_at", + typeName: dax.BaseTypeTimestamp, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds), + }, + }, + { + planner: p, + name: "updated_at", + typeName: dax.BaseTypeTimestamp, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds), + }, + }, + }, + description: "system table for functions", + } + // call next on our iterator to create the table + _, err := iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + } + return nil +} + +func (p *ExecutionPlanner) getFunctionByName(name string) (*functionSystemObject, error) { + err := p.ensureFunctionsSystemTableExists() + if err != nil { + return nil, err + } + + tbl, err := p.schemaAPI.TableByName(context.Background(), "fb_functions") + if err != nil { + return nil, sql3.NewErrTableNotFound(0, 0, "fb_functions") + } + + cols := make([]string, len(tbl.Fields)) + for i, c := range tbl.Fields { + cols[i] = string(c.Name) + } + + iter := &tableScanRowIter{ + planner: p, + tableName: "fb_functions", + columns: cols, + predicate: newBinOpPlanExpression( + newQualifiedRefPlanExpression("fb_functions", "_id", 0, parser.NewDataTypeString()), + parser.EQ, + newStringLiteralPlanExpression(name), + parser.NewDataTypeBool(), + ), + topExpr: nil, + } + + row, err := iter.Next(context.Background()) + if err != nil { + if err == types.ErrNoMoreRows { + // view does not exist + return nil, nil + } + return nil, err + } + + return &functionSystemObject{ + name: row[1].(string), + language: row[2].(string), + body: row[3].(string), + }, nil +} + +func (p *ExecutionPlanner) insertFunction(function *functionSystemObject) error { + err := p.ensureFunctionsSystemTableExists() + if err != nil { + return err + } + + createTime := time.Now().UTC() + + iter := &insertRowIter{ + planner: p, + tableName: "fb_functions", + targetColumns: []*qualifiedRefPlanExpression{ + newQualifiedRefPlanExpression("fb_functions", "_id", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_functions", "name", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_functions", "language", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_functions", "body", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_functions", "owner", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_functions", "updated_by", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_functions", "created_at", 0, parser.NewDataTypeTimestamp()), + newQualifiedRefPlanExpression("fb_functions", "updated_at", 0, parser.NewDataTypeTimestamp()), + }, + insertValues: [][]types.PlanExpression{ + { + newStringLiteralPlanExpression(function.name), + newStringLiteralPlanExpression(function.name), + newStringLiteralPlanExpression(function.language), + newStringLiteralPlanExpression(function.body), + newStringLiteralPlanExpression(""), + newStringLiteralPlanExpression(""), + newTimestampLiteralPlanExpression(createTime), + newTimestampLiteralPlanExpression(createTime), + }, + }, + } + _, err = iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + return nil +} + +func (p *ExecutionPlanner) updateFunction(function *functionSystemObject) error { + err := p.ensureFunctionsSystemTableExists() + if err != nil { + return err + } + + updateTime := time.Now().UTC() + + iter := &insertRowIter{ + planner: p, + tableName: "fb_functions", + targetColumns: []*qualifiedRefPlanExpression{ + newQualifiedRefPlanExpression("fb_functions", "_id", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_functions", "language", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_functions", "body", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_functions", "updated_by", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_functions", "updated_at", 0, parser.NewDataTypeTimestamp()), + }, + insertValues: [][]types.PlanExpression{ + { + newStringLiteralPlanExpression(function.name), + newStringLiteralPlanExpression(function.language), + newStringLiteralPlanExpression(function.body), + newStringLiteralPlanExpression(""), + newTimestampLiteralPlanExpression(updateTime), + }, + }, + } + _, err = iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + return nil +} + +func (p *ExecutionPlanner) deleteFunction(functionName string) error { + err := p.ensureFunctionsSystemTableExists() + if err != nil { + return err + } + + iter := &filteredDeleteRowIter{ + planner: p, + tableName: "fb_functions", + filter: newBinOpPlanExpression( + newQualifiedRefPlanExpression("fb_functions", "_id", 0, parser.NewDataTypeString()), + parser.EQ, + newStringLiteralPlanExpression(functionName), + parser.NewDataTypeBool(), + ), + } + _, err = iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + return nil +} + +func (p *ExecutionPlanner) ensureModelsSystemTableExists() error { + _, err := p.schemaAPI.TableByName(context.Background(), "fb_models") + if err != nil { + if !isTableNotFoundError(err) { + return err + } + // create table fb_models ( + // _id string + // name string + // status string + // model_type string + // labels string --this is an array of string, we'll store it as a json object until we have string[] type in sql + // input_columns string --this is an array for string, we'll store it as a json object until we have string[] type in sql + // owner string + // updated_by string + // created_at timestamp + // updated_at timestamp + // ); + + // if it doesn't, create it by making the appropriate iterator + iter := &createTableRowIter{ + planner: p, + tableName: "fb_models", + failIfExists: false, + isKeyed: true, + keyPartitions: 0, + columns: []*createTableField{ + { + planner: p, + name: "name", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "status", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "model_type", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "labels", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "input_columns", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "owner", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "updated_by", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "created_at", + typeName: dax.BaseTypeTimestamp, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds), + }, + }, + { + planner: p, + name: "updated_at", + typeName: dax.BaseTypeTimestamp, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds), + }, + }, + }, + description: "system table for models", + } + // call next on our iterator to create the table + _, err := iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + } + return nil +} + +func (p *ExecutionPlanner) ensureModelDataSystemTableExists() error { + _, err := p.schemaAPI.TableByName(context.Background(), "fb_model_data") + if err != nil { + if !isTableNotFoundError(err) { + return err + } + // create table fb_model_data ( + // _id string + // model_id string + // data string + // ); + + // if it doesn't, create it by making the appropriate iterator + iter := &createTableRowIter{ + planner: p, + tableName: "fb_model_data", + failIfExists: false, + isKeyed: true, + keyPartitions: 0, + columns: []*createTableField{ + { + planner: p, + name: "model_id", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "data", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + }, + description: "system table for model data", + } + // call next on our iterator to create the table + _, err := iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + } + return nil +} + +func (p *ExecutionPlanner) getModelByName(name string) (*modelSystemObject, error) { + err := p.ensureModelsSystemTableExists() + if err != nil { + return nil, err + } + + tbl, err := p.schemaAPI.TableByName(context.Background(), "fb_models") + if err != nil { + return nil, sql3.NewErrTableNotFound(0, 0, "fb_models") + } + + cols := make([]string, len(tbl.Fields)) + for i, c := range tbl.Fields { + cols[i] = string(c.Name) + } + + iter := &tableScanRowIter{ + planner: p, + tableName: "fb_models", + columns: cols, + predicate: newBinOpPlanExpression( + newQualifiedRefPlanExpression("fb_models", "_id", 0, parser.NewDataTypeString()), + parser.EQ, + newStringLiteralPlanExpression(name), + parser.NewDataTypeBool(), + ), + topExpr: nil, + } + + row, err := iter.Next(context.Background()) + if err != nil { + if err == types.ErrNoMoreRows { + // model does not exist + return nil, nil + } + return nil, err + } + + labels := make([]string, 0) + err = json.Unmarshal([]byte(row[4].(string)), &labels) + if err != nil { + return nil, err + } + + inputColumns := make([]string, 0) + err = json.Unmarshal([]byte(row[5].(string)), &inputColumns) + if err != nil { + return nil, err + } + + return &modelSystemObject{ + name: row[1].(string), + status: row[2].(string), + modelType: row[3].(string), + labels: labels, + inputColumns: inputColumns, + }, nil +} + +func (p *ExecutionPlanner) insertModel(model *modelSystemObject) error { + err := p.ensureModelsSystemTableExists() + if err != nil { + return err + } + + createTime := time.Now().UTC() + + labelJson, err := json.Marshal(model.labels) + if err != nil { + return err + } + + inputColumnsJson, err := json.Marshal(model.inputColumns) + if err != nil { + return err + } + + iter := &insertRowIter{ + planner: p, + tableName: "fb_models", + targetColumns: []*qualifiedRefPlanExpression{ + newQualifiedRefPlanExpression("fb_models", "_id", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "name", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "status", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "model_type", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "labels", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "input_columns", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "owner", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "updated_by", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "created_at", 0, parser.NewDataTypeTimestamp()), + newQualifiedRefPlanExpression("fb_models", "updated_at", 0, parser.NewDataTypeTimestamp()), + }, + insertValues: [][]types.PlanExpression{ + { + newStringLiteralPlanExpression(model.name), + newStringLiteralPlanExpression(model.name), + newStringLiteralPlanExpression(model.status), + newStringLiteralPlanExpression(model.modelType), + newStringLiteralPlanExpression(string(labelJson)), + newStringLiteralPlanExpression(string(inputColumnsJson)), + newStringLiteralPlanExpression(""), + newStringLiteralPlanExpression(""), + newTimestampLiteralPlanExpression(createTime), + newTimestampLiteralPlanExpression(createTime), + }, + }, + } + _, err = iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + return nil +} + +func (p *ExecutionPlanner) updateModel(model *modelSystemObject) error { + err := p.ensureModelsSystemTableExists() + if err != nil { + return err + } + + updateTime := time.Now().UTC() + + labelJson, err := json.Marshal(model.labels) + if err != nil { + return err + } + + inputColumnsJson, err := json.Marshal(model.inputColumns) + if err != nil { + return err + } + + iter := &insertRowIter{ + planner: p, + tableName: "fb_models", + targetColumns: []*qualifiedRefPlanExpression{ + newQualifiedRefPlanExpression("fb_models", "_id", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "status", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "model_type", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "labels", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "input_columns", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "updated_by", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_models", "updated_at", 0, parser.NewDataTypeTimestamp()), + }, + insertValues: [][]types.PlanExpression{ + { + newStringLiteralPlanExpression(model.name), + newStringLiteralPlanExpression(model.status), + newStringLiteralPlanExpression(model.modelType), + newStringLiteralPlanExpression(string(labelJson)), + newStringLiteralPlanExpression(string(inputColumnsJson)), + newStringLiteralPlanExpression(""), + newTimestampLiteralPlanExpression(updateTime), + }, + }, + } + _, err = iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + return nil +} + +func (p *ExecutionPlanner) deleteModel(modelName string) error { + err := p.ensureModelsSystemTableExists() + if err != nil { + return err + } + + err = p.ensureModelDataSystemTableExists() + if err != nil { + return err + } + + iter := &filteredDeleteRowIter{ + planner: p, + tableName: "fb_models", + filter: newBinOpPlanExpression( + newQualifiedRefPlanExpression("fb_models", "_id", 0, parser.NewDataTypeString()), + parser.EQ, + newStringLiteralPlanExpression(modelName), + parser.NewDataTypeBool(), + ), + } + _, err = iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + + iter = &filteredDeleteRowIter{ + planner: p, + tableName: "fb_model_data", + filter: newBinOpPlanExpression( + newQualifiedRefPlanExpression("fb_model_data", "model_id", 0, parser.NewDataTypeString()), + parser.EQ, + newStringLiteralPlanExpression(modelName), + parser.NewDataTypeBool(), + ), + } + _, err = iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + + return nil +} diff --git a/sql3/planner/userdefinedfunctions.go b/sql3/planner/userdefinedfunctions.go new file mode 100644 index 000000000..8b0935c08 --- /dev/null +++ b/sql3/planner/userdefinedfunctions.go @@ -0,0 +1,71 @@ +package planner + +import ( + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" +) + +func (p *ExecutionPlanner) analyzeUserDefinedFunction(call *parser.Call, scope parser.Statement, function *functionSystemObject) (parser.Expr, error) { + // TODO(pok) removing user defined functions for now + // // hard code to 1 string parameter + // if len(call.Args) != 1 { + // return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 2, len(call.Args)) + // } + + // // arg 1 + // argType := parser.NewDataTypeString() + // if !typesAreAssignmentCompatible(argType, call.Args[0].DataType()) { + // return nil, sql3.NewErrParameterTypeMistmatch(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Args[0].DataType().TypeDescription(), argType.TypeDescription()) + // } + + // //return string + // call.ResultDataType = parser.NewDataTypeString() + + // return call, nil + return nil, sql3.NewErrUnsupported(0, 0, false, "user defined functions") +} + +func (n *callPlanExpression) evaluateUserDefinedFunction(currentRow []interface{}) (interface{}, error) { + // TODO(pok) removing what effectively is a remote code exploit + // we will come back to this to add sql udfs and external code later + + // argEval, err := n.args[0].Evaluate(currentRow) + // if err != nil { + // return nil, err + // } + // // nil if anything is nil + // if argEval == nil { + // return nil, nil + // } + + // //get the value + // coercedArg, err := coerceValue(n.args[0].Type(), parser.NewDataTypeString(), argEval, parser.Pos{Line: 0, Column: 0}) + // if err != nil { + // return nil, err + // } + + // arg, argOk := coercedArg.(string) + // if !argOk { + // return nil, sql3.NewErrInternalf("unable to convert value") + // } + + // // save the body to a temp file + // file, err := os.CreateTemp("", "py-body") + // if err != nil { + // return nil, err + // } + // defer os.Remove(file.Name()) + + // file.Write([]byte(n.udfReference.body)) + + // cmd := exec.Command("python3", file.Name(), arg) + // stdout, err := cmd.Output() + + // if err != nil { + // return nil, err + // } + + // retVal := string(stdout) + // return retVal, nil + return nil, sql3.NewErrUnsupported(0, 0, false, "user defined functions") +} diff --git a/sql3/test/defs/defs.go b/sql3/test/defs/defs.go index 6b309737c..5412758db 100644 --- a/sql3/test/defs/defs.go +++ b/sql3/test/defs/defs.go @@ -39,7 +39,7 @@ var TableTests []TableTest = []TableTest{ subqueryTests, viewTests, - topTests, + topLimitTests, deleteTests, @@ -179,6 +179,8 @@ var TableTests []TableTest = []TableTest{ avgTests, percentileTests, minmaxTests, + corrTests, + varTests, // groupby tests groupByTests, @@ -197,6 +199,10 @@ var TableTests []TableTest = []TableTest{ bulkInsertTable, bulkInsert, + // copy + copyTable, + copyTests, + // bool (batch logic) boolTests, diff --git a/sql3/test/defs/defs_aggregate.go b/sql3/test/defs/defs_aggregate.go index a52698685..0532d3c40 100644 --- a/sql3/test/defs/defs_aggregate.go +++ b/sql3/test/defs/defs_aggregate.go @@ -737,3 +737,178 @@ var minmaxTests = TableTest{ }, }, } + +var corrTests = TableTest{ + Table: tbl( + "corr_test", + srcHdrs( + srcHdr("_id", fldTypeID), + srcHdr("i1", fldTypeInt, "min 0", "max 1000"), + srcHdr("d1", fldTypeDecimal2), + srcHdr("s1", fldTypeString), + srcHdr("id1", fldTypeID), + ), + srcRows( + srcRow(int64(1), int64(10), float64(10), string("foo"), int64(10)), + srcRow(int64(2), int64(10), float64(10), string("foo2"), int64(11)), + srcRow(int64(3), int64(11), float64(11), string("foo34"), int64(12)), + srcRow(int64(4), int64(12), float64(12), string("foo45"), int64(13)), + srcRow(int64(5), int64(12), float64(12), string("foo63"), int64(14)), + srcRow(int64(6), int64(13), float64(13), string("foo22222"), int64(15)), + ), + ), + SQLTests: []SQLTest{ + { + SQLs: sqls( + "SELECT corr(*, i1) AS corr1 FROM corr_test", + ), + ExpErr: "expected right paren, found ','", + }, + { + SQLs: sqls( + "SELECT corr(i1, d1) AS corr1 FROM corr_test", + ), + ExpHdrs: hdrs( + hdr("corr1", featurebase.WireQueryField{ + Type: dax.BaseTypeDecimal + "(6)", + BaseType: dax.BaseTypeDecimal, + TypeInfo: map[string]interface{}{"scale": int64(6)}, + }), + ), + ExpRows: rows( + row(pql.NewDecimal(1000000, 6)), + ), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "SELECT corr(_id, i1) AS corr1 FROM corr_test", + ), + ExpErr: "_id column cannot be used in aggregate function 'corr'", + }, + { + SQLs: sqls( + "SELECT corr(i1) AS corr1 FROM corr_test", + ), + ExpErr: "count of formal parameters (2) does not match count of actual parameters (1)", + }, + { + SQLs: sqls( + "SELECT corr(s1, i1) AS avg_rows FROM corr_test", + ), + ExpErr: "integer, decimal or timestamp expression expected", + }, + { + SQLs: sqls( + "SELECT corr(len(s1), i1) AS corr1 FROM corr_test", + ), + ExpHdrs: hdrs( + hdr("corr1", featurebase.WireQueryField{ + Type: dax.BaseTypeDecimal + "(6)", + BaseType: dax.BaseTypeDecimal, + TypeInfo: map[string]interface{}{"scale": int64(6)}, + }), + ), + ExpRows: rows( + row(pql.NewDecimal(888234, 6)), + ), + Compare: CompareExactUnordered, + }, + }, +} + +var varTests = TableTest{ + Table: tbl( + "var_test", + srcHdrs( + srcHdr("_id", fldTypeID), + srcHdr("i1", fldTypeInt, "min 0", "max 1000"), + srcHdr("d1", fldTypeDecimal2), + srcHdr("s1", fldTypeString), + srcHdr("id1", fldTypeID), + ), + srcRows( + srcRow(int64(1), int64(10), float64(10), string("foo"), int64(10)), + srcRow(int64(2), int64(10), float64(10), string("foo"), int64(11)), + srcRow(int64(3), int64(11), float64(11), string("foo"), int64(12)), + srcRow(int64(4), int64(12), float64(12), string("foo"), int64(13)), + srcRow(int64(5), int64(12), float64(12), string("foo"), int64(14)), + srcRow(int64(6), int64(13), float64(13), string("foo"), int64(15)), + ), + ), + SQLTests: []SQLTest{ + { + SQLs: sqls( + "SELECT var(*) AS var1 FROM var_test", + ), + ExpErr: "column reference expected", + }, + { + SQLs: sqls( + "SELECT var(_id) AS var1 FROM var_test", + ), + ExpErr: "_id column cannot be used in aggregate function 'var'", + }, + { + SQLs: sqls( + "SELECT var(i1, d1) AS var1 FROM var_test", + ), + ExpErr: "count of formal parameters (1) does not match count of actual parameters (2)", + }, + { + SQLs: sqls( + "SELECT var(s1) AS var1 FROM var_test", + ), + ExpErr: "integer, decimal or timestamp expression expected", + }, + { + SQLs: sqls( + "SELECT var(id1) AS var1 FROM var_test", + ), + ExpHdrs: hdrs( + hdr("var1", featurebase.WireQueryField{ + Type: dax.BaseTypeDecimal + "(6)", + BaseType: dax.BaseTypeDecimal, + TypeInfo: map[string]interface{}{"scale": int64(6)}, + }), + ), + ExpRows: rows( + row(pql.NewDecimal(2916666, 6)), + ), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "SELECT var(i1) AS var1 FROM var_test", + "SELECT var(d1) AS var1 FROM var_test", + ), + ExpHdrs: hdrs( + hdr("var1", featurebase.WireQueryField{ + Type: dax.BaseTypeDecimal + "(6)", + BaseType: dax.BaseTypeDecimal, + TypeInfo: map[string]interface{}{"scale": int64(6)}, + }), + ), + ExpRows: rows( + row(pql.NewDecimal(1222222, 6)), + ), + Compare: CompareExactUnordered, + }, + { + SQLs: sqls( + "SELECT var(len(s1)) AS var1 FROM var_test", + ), + ExpHdrs: hdrs( + hdr("var1", featurebase.WireQueryField{ + Type: dax.BaseTypeDecimal + "(6)", + BaseType: dax.BaseTypeDecimal, + TypeInfo: map[string]interface{}{"scale": int64(6)}, + }), + ), + ExpRows: rows( + row(pql.NewDecimal(0, 6)), + ), + Compare: CompareExactUnordered, + }, + }, +} diff --git a/sql3/test/defs/defs_bulkinsert.go b/sql3/test/defs/defs_bulkinsert.go index 57f0b4036..3fcafa852 100644 --- a/sql3/test/defs/defs_bulkinsert.go +++ b/sql3/test/defs/defs_bulkinsert.go @@ -1,6 +1,6 @@ package defs -// join tests +// bulk insert var bulkInsertTable = TableTest{ name: "bulkInsertTable", Table: tbl( diff --git a/sql3/test/defs/defs_copy.go b/sql3/test/defs/defs_copy.go new file mode 100644 index 000000000..3fb3be48d --- /dev/null +++ b/sql3/test/defs/defs_copy.go @@ -0,0 +1,58 @@ +// Copyright 2023 Molecula Corp. All rights reserved. + +package defs + +// copy +var copyTable = TableTest{ + name: "copyTable", + Table: tbl( + "copytest", + srcHdrs( + srcHdr("_id", fldTypeID), + srcHdr("id_col", fldTypeID), + srcHdr("string_col", fldTypeString), + srcHdr("int_col", fldTypeInt), + srcHdr("decimal_col", fldTypeDecimal2), + srcHdr("bool_col", fldTypeBool), + srcHdr("time_col", fldTypeTimestamp), + srcHdr("stringset_col", fldTypeStringSet), + srcHdr("idset_col", fldTypeIDSet), + ), + srcRows( + srcRow(int64(1), int64(10), string("foo"), int64(10), float64(10), bool(false), knownTimestamp(), []string{"foo", "bar"}, []int64{1, 2}), + srcRow(int64(2), int64(11), string("foo1"), int64(11), float64(11), bool(true), knownTimestamp(), []string{"foo1", "bar1"}, []int64{11, 21}), + srcRow(int64(3), int64(12), string("foo2"), int64(12), float64(12), bool(false), knownTimestamp(), []string{"foo2", "bar2"}, []int64{12, 22}), + srcRow(int64(4), int64(13), string("foo3"), int64(13), float64(13), bool(true), knownTimestamp(), []string{"foo3", "bar3"}, []int64{13, 23}), + ), + ), + SQLTests: nil, +} + +var copyTests = TableTest{ + name: "copyTests", + SQLTests: []SQLTest{ + { + name: "copy-no-table-to-table", + SQLs: sqls( + `copy foo to bar;`, + ), + ExpErr: "table or view 'foo' not found", + }, + { + name: "copy-table-to-table-same-name", + SQLs: sqls( + `copy copytest to copytest;`, + ), + ExpErr: "already exists", + }, + { + name: "copy-table-to-table", + SQLs: sqls( + `copy copytest to copytesttwo;`, + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactOrdered, + }, + }, +} diff --git a/sql3/test/defs/defs_top.go b/sql3/test/defs/defs_top.go index f4c61e41a..324cc9c00 100644 --- a/sql3/test/defs/defs_top.go +++ b/sql3/test/defs/defs_top.go @@ -1,7 +1,7 @@ package defs -var topTests = TableTest{ - name: "top-tests", +var topLimitTests = TableTest{ + name: "top-limit-tests", Table: tbl( "skills", srcHdrs( @@ -20,7 +20,25 @@ var topTests = TableTest{ SQLTests: []SQLTest{ { SQLs: sqls( - "select top(1) * from skills where setcontains(skills, 'Marketing Manager');", + "select top(1) * from skills where setcontains(skills, 'Marketing Manager');", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + hdr("bools", fldTypeStringSet), + hdr("bools-exist", fldTypeStringSet), + hdr("id1", fldTypeID), + hdr("skills", fldTypeStringSet), + hdr("titles", fldTypeStringSet), + ), + ExpRows: rows( + row(int64(1), nil, []string{"available_for_hire"}, int64(288), []string{"Marketing Manager"}, []string{"Alumni Relations", "OEM negotiations"}), + ), + Compare: CompareExactUnordered, + SortStringKeys: true, + }, + { + SQLs: sqls( + "select * from skills where setcontains(skills, 'Marketing Manager') limit 1;", ), ExpHdrs: hdrs( hdr("_id", fldTypeID), @@ -55,6 +73,21 @@ var topTests = TableTest{ Compare: CompareExactUnordered, SortStringKeys: true, }, + { + SQLs: sqls( + "select count(*), skills from skills group by skills limit 10;", + ), + ExpHdrs: hdrs( + hdr("", fldTypeInt), + hdr("skills", fldTypeStringSet), + ), + ExpRows: rows( + row(int64(1), string("Marketing Manager")), + row(int64(1), string("Software Engineer I")), + ), + Compare: CompareExactUnordered, + SortStringKeys: true, + }, { SQLs: sqls( "select top(1) count(*) from skills;", @@ -68,5 +101,24 @@ var topTests = TableTest{ Compare: CompareExactUnordered, SortStringKeys: true, }, + { + SQLs: sqls( + "select count(*) from skills limit 1;", + ), + ExpHdrs: hdrs( + hdr("", fldTypeInt), + ), + ExpRows: rows( + row(int64(2)), + ), + Compare: CompareExactUnordered, + SortStringKeys: true, + }, + { + SQLs: sqls( + "select top(1) count(*) from skills limit 1;", + ), + ExpErr: "TOP and LIMIT cannot cannot be used at the same time", + }, }, }