scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

create model, create function... all the goodies (#2264)

Browse source 
* create function, create/drop model; re-introduced limit; added COPY; var(); corr()

* review feedback
This commit is contained in:
Pat Okeeffe 2023-04-04 17:44:29 -05:00 • committed by GitHub
parent c8c88ab0ee
commit 284f62dcb9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 4104 additions and 495 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
dax/test/dax/dax_test.go
View file

@ -144,7 +144,8 @@ func TestDAXIntegration(t *testing.T) {
"testinsert/test-5", // error messages differ
"percentile_test/test-6", // related to TODO in orchestrator.executePercentile
"alterTable/alterTableBadTable", // looks like table does not exist is a different error in DAX
"top-tests/test-1", // don't know why this is failing at all
"top-limit-tests/test-2", // don't know why this is failing at all
"top-limit-tests/test-3", // don't know why this is failing at all
"delete_tests",
"viewtests/drop-view", // drop view does a delete
"viewtests/drop-view-if-exists-after-drop",

2
go.mod
View file

@ -137,6 +137,7 @@ require (
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d // indirect
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e // indirect
github.com/tinylib/msgp v1.1.2 // indirect
gonum.org/v1/gonum v0.11.0 // indirect
)
require (
@ -197,6 +198,7 @@ require (
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/sajari/regression v1.0.1
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/soheilhy/cmux v0.1.5 // indirect
github.com/spf13/afero v1.6.0 // indirect

3
go.sum
View file

@ -1056,6 +1056,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/sajari/regression v1.0.1 h1:iTVc6ZACGCkoXC+8NdqH5tIreslDTT/bXxT6OmHR5PE=
github.com/sajari/regression v1.0.1/go.mod h1:NeG/XTW1lYfGY7YV/Z0nYDV/RGh3wxwd1yW46835flM=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@ -1684,6 +1686,7 @@ golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNq
gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=

45
sql3/errors.go
View file

@ -13,10 +13,13 @@ const (
ErrUnsupported errors.Code = "ErrUnsupported"
ErrCacheKeyNotFound errors.Code = "ErrCacheKeyNotFound"
ErrDuplicateColumn errors.Code = "ErrDuplicateColumn"
ErrUnknownType errors.Code = "ErrUnknownType"
ErrUnknownIdentifier errors.Code = "ErrUnknownIdentifier"
// syntax/semantic errors
ErrDuplicateColumn errors.Code = "ErrDuplicateColumn"
ErrUnknownType errors.Code = "ErrUnknownType"
ErrUnknownIdentifier errors.Code = "ErrUnknownIdentifier"
ErrTopLimitCannotCoexist errors.Code = "ErrTopLimitCannotCoexist"
// type related errors
ErrTypeIncompatibleWithBitwiseOperator errors.Code = "ErrTypeIncompatibleWithBitwiseOperator"
ErrTypeIncompatibleWithLogicalOperator errors.Code = "ErrTypeIncompatibleWithLogicalOperator"
ErrTypeIncompatibleWithEqualityOperator errors.Code = "ErrTypeIncompatibleWithEqualityOperator"
@ -40,8 +43,6 @@ const (
ErrTimeQuantumExpressionExpected errors.Code = "ErrTimeQuantumExpressionExpected"
ErrSingleRowExpected errors.Code = "ErrSingleRowExpected"
// type related errors
// decimal
ErrDecimalScaleExpected errors.Code = "ErrDecimalScaleExpected"
@ -94,6 +95,9 @@ const (
ErrViewExists errors.Code = "ErrViewExists"
ErrViewNotFound errors.Code = "ErrViewNotFound"
ErrModelExists errors.Code = "ErrModelExists"
ErrModelNotFound errors.Code = "ErrModelNotFound"
ErrBadColumnConstraint errors.Code = "ErrBadColumnConstraint"
ErrConflictingColumnConstraint errors.Code = "ErrConflictingColumnConstraint"
@ -141,6 +145,9 @@ const (
ErrInvalidDatetimePart errors.Code = "ErrInvalidDatetimePart"
ErrOutputValueOutOfRange errors.Code = "ErrOutputValueOutOfRange"
ErrDivideByZero errors.Code = "ErrDivideByZero"
// remote execution
ErrRemoteUnauthorized errors.Code = "ErrRemoteUnauthorized"
)
func NewErrDuplicateColumn(line int, col int, column string) error {
@ -164,6 +171,13 @@ func NewErrUnknownIdentifier(line int, col int, ident string) error {
)
}
func NewErrErrTopLimitCannotCoexist(line int, col int) error {
return errors.New(
ErrTopLimitCannotCoexist,
fmt.Sprintf("[%d:%d] TOP and LIMIT cannot cannot be used at the same time (TOP will be deprecated in a future release)", line, col),
)
}
func NewErrInternal(msg string) error {
preamble := "internal error"
_, filename, line, ok := runtime.Caller(1)
@ -647,6 +661,20 @@ func NewErrViewExists(line, col int, viewName string) error {
)
}
func NewErrModelNotFound(line, col int, viewName string) error {
return errors.New(
ErrModelNotFound,
fmt.Sprintf("[%d:%d] model '%s' not found", line, col, viewName),
)
}
func NewErrModelExists(line, col int, viewName string) error {
return errors.New(
ErrModelExists,
fmt.Sprintf("[%d:%d] model '%s' already exists", line, col, viewName),
)
}
func NewErrBadColumnConstraint(line, col int, constraint, columnType string) error {
return errors.New(
ErrBadColumnConstraint,
@ -876,3 +904,10 @@ func NewErrDivideByZero(line, col int) error {
fmt.Sprintf("[%d:%d] divisor is equal to zero", line, col),
)
}
func NewErrRemoteUnauthorized(line, col int, remoteUrl string) error {
return errors.New(
ErrRemoteUnauthorized,
fmt.Sprintf("unauthorized on remote server '%s'", remoteUrl),
)
}

250
sql3/parser/ast.go
View file

@ -34,11 +34,13 @@ func (*CaseExpr) node() {}
func (*CastExpr) node() {}
func (*CheckConstraint) node() {}
func (*ColumnDefinition) node() {}
func (*CopyStatement) node() {}
func (*CommitStatement) node() {}
func (*CreateDatabaseStatement) node() {}
func (*CreateIndexStatement) node() {}
func (*CreateTableStatement) node() {}
func (*CreateFunctionStatement) node() {}
func (*CreateModelStatement) node() {}
func (*CreateViewStatement) node() {}
func (*DateLit) node() {}
func (*DefaultConstraint) node() {}
@ -48,6 +50,7 @@ func (*DropIndexStatement) node() {}
func (*DropTableStatement) node() {}
func (*DropFunctionStatement) node() {}
func (*DropViewStatement) node() {}
func (*DropModelStatement) node() {}
func (*Exists) node() {}
func (*ExplainStatement) node() {}
func (*ExprList) node() {}
@ -73,12 +76,14 @@ func (*OnConstraint) node() {}
func (*OrderingTerm) node() {}
func (*OverClause) node() {}
func (*ParenExpr) node() {}
func (*PredictStatement) node() {}
func (*SetLiteralExpr) node() {}
func (*ParenSource) node() {}
func (*PrimaryKeyConstraint) node() {}
func (*QualifiedRef) node() {}
func (*QualifiedTableName) node() {}
func (*Range) node() {}
func (*ReturnStatement) node() {}
func (*ReleaseStatement) node() {}
func (*ResultColumn) node() {}
func (*RollbackStatement) node() {}
@ -111,6 +116,7 @@ func (*AlterTableStatement) stmt() {}
func (*AlterViewStatement) stmt() {}
func (*AnalyzeStatement) stmt() {}
func (*BeginStatement) stmt() {}
func (*CopyStatement) stmt() {}
func (*BulkInsertStatement) stmt() {}
func (*ShowDatabasesStatement) stmt() {}
func (*ShowTablesStatement) stmt() {}
@ -121,6 +127,7 @@ func (*CreateDatabaseStatement) stmt() {}
func (*CreateIndexStatement) stmt() {}
func (*CreateTableStatement) stmt() {}
func (*CreateFunctionStatement) stmt() {}
func (*CreateModelStatement) stmt() {}
func (*CreateViewStatement) stmt() {}
func (*DeleteStatement) stmt() {}
func (*DropDatabaseStatement) stmt() {}
@ -128,9 +135,12 @@ func (*DropIndexStatement) stmt() {}
func (*DropTableStatement) stmt() {}
func (*DropFunctionStatement) stmt() {}
func (*DropViewStatement) stmt() {}
func (*DropModelStatement) stmt() {}
func (*PredictStatement) stmt() {}
func (*ExplainStatement) stmt() {}
func (*InsertStatement) stmt() {}
func (*ReleaseStatement) stmt() {}
func (*ReturnStatement) stmt() {}
func (*RollbackStatement) stmt() {}
func (*SavepointStatement) stmt() {}
func (*SelectStatement) stmt() {}
@ -177,6 +187,8 @@ func CloneStatement(stmt Statement) Statement {
return stmt.Clone()
case *DropViewStatement:
return stmt.Clone()
case *DropModelStatement:
return stmt.Clone()
case *ExplainStatement:
return stmt.Clone()
case *InsertStatement:
@ -2910,6 +2922,35 @@ func (s *DropViewStatement) String() string {
return buf.String()
}
type DropModelStatement struct {
Drop Pos // position of DROP keyword
Model Pos // position of MODEL keyword
If Pos // position of IF keyword
IfExists Pos // position of EXISTS keyword after IF
Name *Ident // view name
}
// Clone returns a deep copy of s.
func (s *DropModelStatement) Clone() *DropModelStatement {
if s == nil {
return nil
}
other := *s
other.Name = s.Name.Clone()
return &other
}
// String returns the string representation of the statement.
func (s *DropModelStatement) String() string {
var buf bytes.Buffer
buf.WriteString("DROP MODEL")
if s.IfExists.IsValid() {
buf.WriteString(" IF EXISTS")
}
fmt.Fprintf(&buf, " %s", s.Name.String())
return buf.String()
}
type CreateIndexStatement struct {
Create Pos // position of CREATE keyword
Unique Pos // position of optional UNIQUE keyword
@ -2998,6 +3039,11 @@ func (s *DropIndexStatement) String() string {
return buf.String()
}
type FunctionOptionDefinition struct {
Name *Ident // option name
OptionExpr Expr // option expression
}
type ParameterDefinition struct {
Name *Variable // parameter name
Type *Type // data type
@ -3015,8 +3061,11 @@ type CreateFunctionStatement struct {
Parameters []*ParameterDefinition // parameters
Rparen Pos // position of parameter RParen
Returns Pos // position of RETURNS keyword
ReturnDef *ParameterDefinition // return def
Returns Pos // position of RETURNS keyword
ReturnType *Type // return def
With Pos // position of WITH keyword
Options []*FunctionOptionDefinition // options
As Pos // position of AS keyword
@ -3057,7 +3106,21 @@ func (s *CreateFunctionStatement) String() string {
}
buf.WriteString(" RETURNS ")
fmt.Fprintf(&buf, "%s %s", s.ReturnDef.Name, s.ReturnDef.Type.Name)
fmt.Fprintf(&buf, "%s", s.ReturnType.String())
if s.With.IsValid() {
buf.WriteString(" WITH ")
if len(s.Options) > 0 {
buf.WriteString(" (")
for idx, p := range s.Options {
if idx > 0 {
buf.WriteString(", ")
}
fmt.Fprintf(&buf, "%s %s", p.Name.Name, p.OptionExpr.String())
}
buf.WriteString(")")
}
}
buf.WriteString(" AS BEGIN")
for i := range s.Body {
@ -3096,6 +3159,86 @@ func (s *DropFunctionStatement) String() string {
return buf.String()
}
type ReturnStatement struct {
Return Pos // position of RETURN keyword
ReturnExpr Expr // what we are returning
}
// Clone returns a deep copy of s.
func (s *ReturnStatement) Clone() *ReturnStatement {
if s == nil {
return nil
}
other := *s
other.ReturnExpr = CloneExpr(s.ReturnExpr)
return &other
}
func (s *ReturnStatement) String() string {
var buf bytes.Buffer
buf.WriteString("RETURN")
fmt.Fprintf(&buf, " %s", s.ReturnExpr.String())
return buf.String()
}
type ModelOptionDefinition struct {
Name *Ident // option name
OptionExpr Expr // option expression
}
type CreateModelStatement struct {
Create Pos // position of CREATE keyword
Model Pos // position of MODEL keyword
If Pos // position of IF keyword
IfNot Pos // position of NOT keyword after IF
IfNotExists Pos // position of EXISTS keyword after IF NOT
Name *Ident // model name
With Pos // position of WITH keyword
Options []*ModelOptionDefinition // options
As Pos // position of AS keyword
ModelQuery *SelectStatement // model query
}
// Clone returns a deep copy of s.
func (s *CreateModelStatement) Clone() *CreateModelStatement {
if s == nil {
return nil
}
other := *s
other.Name = s.Name.Clone()
other.ModelQuery = s.ModelQuery.Clone()
return &other
}
// String returns the string representation of the statement.
func (s *CreateModelStatement) String() string {
var buf bytes.Buffer
buf.WriteString("CREATE MODEL")
if s.IfNotExists.IsValid() {
buf.WriteString(" IF NOT EXISTS")
}
fmt.Fprintf(&buf, " %s", s.Name.String())
if len(s.Options) > 0 {
buf.WriteString(" (")
for idx, p := range s.Options {
if idx > 0 {
buf.WriteString(", ")
}
fmt.Fprintf(&buf, "%s %s", p.Name.Name, p.OptionExpr.String())
}
buf.WriteString(")")
}
buf.WriteString(" AS ")
buf.WriteString(s.ModelQuery.String())
return buf.String()
}
type BulkInsertMapDefinition struct {
Name *Ident // map name
Type *Type // data type
@ -3645,6 +3788,77 @@ func (c *IndexedColumn) String() string {
return c.X.String()
}
type CopyStatement struct {
Copy Pos // position of COPY keyword
Source Source // source table
To Pos // position of TO keyword
TargetName *Ident // target table name
Where Pos // position of WHERE keyword
WhereExpr Expr // where clause expression
With Pos // position of WITH keyword
Url Expr // url for target server
ApiKey Expr // apikey for target server
}
func (c *CopyStatement) Clone() *CopyStatement {
if c == nil {
return nil
}
other := *c
other.Source = CloneSource(c.Source)
other.TargetName = c.TargetName.Clone()
other.WhereExpr = CloneExpr(c.WhereExpr)
other.Url = CloneExpr(c.Url)
other.ApiKey = CloneExpr(c.ApiKey)
return &other
}
func (c *CopyStatement) String() string {
var buf bytes.Buffer
fmt.Fprintf(&buf, "COPY %s to %s", c.Source.String(), c.TargetName.String())
if c.WhereExpr != nil {
fmt.Fprintf(&buf, " WHERE %s", c.WhereExpr.String())
}
if c.With.IsValid() {
fmt.Fprintf(&buf, " WITH")
if c.Url != nil {
fmt.Fprintf(&buf, " URL %s", c.Url.String())
}
if c.ApiKey != nil {
fmt.Fprintf(&buf, " APIKEY %s", c.Url.String())
}
}
return buf.String()
}
type PredictStatement struct {
Predict Pos // position of PREDICT keyword
Using Pos // position of USING keyword
ModelName *Ident // model name
InputQuery *SelectStatement // input query
}
func (c *PredictStatement) Clone() *PredictStatement {
if c == nil {
return nil
}
other := *c
other.ModelName = c.ModelName.Clone()
other.InputQuery = c.InputQuery.Clone()
return &other
}
func (c *PredictStatement) String() string {
var buf bytes.Buffer
fmt.Fprintf(&buf, "PREDICT USING %s", c.ModelName.String())
fmt.Fprintf(&buf, " %s", c.InputQuery.String())
return buf.String()
}
type SelectStatement struct {
WithClause *WithClause // clause containing CTEs
@ -3685,6 +3899,8 @@ type SelectStatement struct {
OrderBy Pos // position of BY keyword after ORDER
OrderingTerms []*OrderingTerm // terms of ORDER BY clause
Limit Pos // position of LIMIT keyword
LimitExpr Expr // LIMIT expr
}
// Clone returns a deep copy of s.
@ -3694,7 +3910,6 @@ func (s *SelectStatement) Clone() *SelectStatement {
}
other := *s
other.WithClause = s.WithClause.Clone()
//other.ValueLists = cloneExprLists(s.ValueLists)
other.TopExpr = CloneExpr(s.TopExpr)
other.Columns = cloneResultColumns(s.Columns)
other.Source = CloneSource(s.Source)
@ -3704,6 +3919,7 @@ func (s *SelectStatement) Clone() *SelectStatement {
other.Windows = cloneWindows(s.Windows)
other.Compound = s.Compound.Clone()
other.OrderingTerms = cloneOrderingTerms(s.OrderingTerms)
other.LimitExpr = CloneExpr(s.LimitExpr)
return &other
}
@ -3742,29 +3958,10 @@ func (s *SelectStatement) String() string {
buf.WriteString(" ")
}
/*if len(s.ValueLists) > 0 {
buf.WriteString("VALUES ")
for i, exprs := range s.ValueLists {
if i != 0 {
buf.WriteString(", ")
}
buf.WriteString("(")
for j, expr := range exprs.Exprs {
if j != 0 {
buf.WriteString(", ")
}
buf.WriteString(expr.String())
}
buf.WriteString(")")
}
} else {*/
buf.WriteString("SELECT ")
if s.Distinct.IsValid() {
buf.WriteString("DISTINCT ")
} //else if s.All.IsValid() {
// buf.WriteString("ALL ")
//}
}
if s.Top.IsValid() {
fmt.Fprintf(&buf, "TOP(%s) ", s.TopExpr.String())
}
@ -3810,7 +4007,6 @@ func (s *SelectStatement) String() string {
buf.WriteString(window.String())
}
}
// }
// Write compound operator.
if s.Compound != nil {
@ -3840,6 +4036,10 @@ func (s *SelectStatement) String() string {
}
}
if s.Limit.IsValid() {
fmt.Fprintf(&buf, "LIMIT %s", s.LimitExpr.String())
}
return buf.String()
}

128
sql3/parser/ast_test.go
View file

@ -404,11 +404,8 @@ func TestCreateFunctionStatement_String(t *testing.T) {
Type: &parser.Type{Name: &parser.Ident{Name: "int"}},
},
},
ReturnDef: &parser.ParameterDefinition{
Name: &parser.Variable{Name: "@scalar"},
Type: &parser.Type{Name: &parser.Ident{Name: "int"}},
},
}, `CREATE FUNCTION func (@param1 int) RETURNS @scalar int AS BEGIN END`)
ReturnType: &parser.Type{Name: &parser.Ident{Name: "int"}},
}, `CREATE FUNCTION func (@param1 int) RETURNS int AS BEGIN END`)
AssertStatementStringer(t, &parser.CreateFunctionStatement{
IfNotExists: pos(0),
@ -419,11 +416,8 @@ func TestCreateFunctionStatement_String(t *testing.T) {
Type: &parser.Type{Name: &parser.Ident{Name: "int"}},
},
},
ReturnDef: &parser.ParameterDefinition{
Name: &parser.Variable{Name: "@scalar"},
Type: &parser.Type{Name: &parser.Ident{Name: "int"}},
},
}, `CREATE FUNCTION IF NOT EXISTS func (@param1 int) RETURNS @scalar int AS BEGIN END`)
ReturnType: &parser.Type{Name: &parser.Ident{Name: "int"}},
}, `CREATE FUNCTION IF NOT EXISTS func (@param1 int) RETURNS int AS BEGIN END`)
}
func TestCreateViewStatement_String(t *testing.T) {
@ -866,34 +860,34 @@ func TestSelectStatement_String(t *testing.T) {
},
}, `SELECT * FROM (SELECT *)`)
AssertStatementStringer(t, &parser.SelectStatement{
Columns: []*parser.ResultColumn{{Star: pos(0)}},
Source: &parser.QualifiedTableName{Name: &parser.Ident{Name: "tbl"}},
Windows: []*parser.Window{
{
Name: &parser.Ident{Name: "win1"},
Definition: &parser.WindowDefinition{
Base: &parser.Ident{Name: "base"},
Partitions: []parser.Expr{&parser.Ident{Name: "x"}, &parser.Ident{Name: "y"}},
OrderingTerms: []*parser.OrderingTerm{
{X: &parser.Ident{Name: "x"}, Asc: pos(0), NullsFirst: pos(0)},
{X: &parser.Ident{Name: "y"}, Desc: pos(0), NullsLast: pos(0)},
},
Frame: &parser.FrameSpec{
Range: pos(0),
UnboundedX: pos(0),
PrecedingX: pos(0),
},
},
},
{
Name: &parser.Ident{Name: "win2"},
Definition: &parser.WindowDefinition{
Base: &parser.Ident{Name: "base2"},
},
},
},
}, `SELECT * FROM tbl WINDOW win1 AS (base PARTITION BY x, y ORDER BY x ASC NULLS FIRST, y DESC NULLS LAST RANGE UNBOUNDED PRECEDING), win2 AS (base2)`)
// AssertStatementStringer(t, &parser.SelectStatement{
// Columns: []*parser.ResultColumn{{Star: pos(0)}},
// Source: &parser.QualifiedTableName{Name: &parser.Ident{Name: "tbl"}},
// Windows: []*parser.Window{
// {
// Name: &parser.Ident{Name: "win1"},
// Definition: &parser.WindowDefinition{
// Base: &parser.Ident{Name: "base"},
// Partitions: []parser.Expr{&parser.Ident{Name: "x"}, &parser.Ident{Name: "y"}},
// OrderingTerms: []*parser.OrderingTerm{
// {X: &parser.Ident{Name: "x"}, Asc: pos(0), NullsFirst: pos(0)},
// {X: &parser.Ident{Name: "y"}, Desc: pos(0), NullsLast: pos(0)},
// },
// Frame: &parser.FrameSpec{
// Range: pos(0),
// UnboundedX: pos(0),
// PrecedingX: pos(0),
// },
// },
// },
// {
// Name: &parser.Ident{Name: "win2"},
// Definition: &parser.WindowDefinition{
// Base: &parser.Ident{Name: "base2"},
// },
// },
// },
// }, `SELECT * FROM tbl WINDOW win1 AS (base PARTITION BY x, y ORDER BY x ASC NULLS FIRST, y DESC NULLS LAST RANGE UNBOUNDED PRECEDING), win2 AS (base2)`)
// AssertStatementStringer(t, &sql.SelectStatement{
// WithClause: &sql.WithClause{
@ -914,38 +908,38 @@ func TestSelectStatement_String(t *testing.T) {
// },
// }, `WITH "cte" ("x", "y") AS (SELECT *) VALUES (1, 2), (3, 4)`)
AssertStatementStringer(t, &parser.SelectStatement{
Columns: []*parser.ResultColumn{{Star: pos(0)}},
Union: pos(0),
Compound: &parser.SelectStatement{
Columns: []*parser.ResultColumn{{Star: pos(0)}},
},
}, `SELECT * UNION SELECT *`)
// AssertStatementStringer(t, &parser.SelectStatement{
// Columns: []*parser.ResultColumn{{Star: pos(0)}},
// Union: pos(0),
// Compound: &parser.SelectStatement{
// Columns: []*parser.ResultColumn{{Star: pos(0)}},
// },
// }, `SELECT * UNION SELECT *`)
AssertStatementStringer(t, &parser.SelectStatement{
Columns: []*parser.ResultColumn{{Star: pos(0)}},
Union: pos(0),
UnionAll: pos(0),
Compound: &parser.SelectStatement{
Columns: []*parser.ResultColumn{{Star: pos(0)}},
},
}, `SELECT * UNION ALL SELECT *`)
// AssertStatementStringer(t, &parser.SelectStatement{
// Columns: []*parser.ResultColumn{{Star: pos(0)}},
// Union: pos(0),
// UnionAll: pos(0),
// Compound: &parser.SelectStatement{
// Columns: []*parser.ResultColumn{{Star: pos(0)}},
// },
// }, `SELECT * UNION ALL SELECT *`)
AssertStatementStringer(t, &parser.SelectStatement{
Columns: []*parser.ResultColumn{{Star: pos(0)}},
Intersect: pos(0),
Compound: &parser.SelectStatement{
Columns: []*parser.ResultColumn{{Star: pos(0)}},
},
}, `SELECT * INTERSECT SELECT *`)
// AssertStatementStringer(t, &parser.SelectStatement{
// Columns: []*parser.ResultColumn{{Star: pos(0)}},
// Intersect: pos(0),
// Compound: &parser.SelectStatement{
// Columns: []*parser.ResultColumn{{Star: pos(0)}},
// },
// }, `SELECT * INTERSECT SELECT *`)
AssertStatementStringer(t, &parser.SelectStatement{
Columns: []*parser.ResultColumn{{Star: pos(0)}},
Except: pos(0),
Compound: &parser.SelectStatement{
Columns: []*parser.ResultColumn{{Star: pos(0)}},
},
}, `SELECT * EXCEPT SELECT *`)
// AssertStatementStringer(t, &parser.SelectStatement{
// Columns: []*parser.ResultColumn{{Star: pos(0)}},
// Except: pos(0),
// Compound: &parser.SelectStatement{
// Columns: []*parser.ResultColumn{{Star: pos(0)}},
// },
// }, `SELECT * EXCEPT SELECT *`)
AssertStatementStringer(t, &parser.SelectStatement{
Columns: []*parser.ResultColumn{{Star: pos(0)}},

546
sql3/parser/parser.go
View file

@ -45,10 +45,10 @@ func (p *Parser) ParseStatement() (stmt Statement, err error) {
switch tok := p.peek(); tok {
case EOF:
return nil, io.EOF
//case EXPLAIN:
// if stmt, err = p.parseExplainStatement(); err != nil {
// return stmt, err
// }
case EXPLAIN:
if stmt, err = p.parseExplainStatement(); err != nil {
return stmt, err
}
default:
if stmt, err = p.parseNonExplainStatement(); err != nil {
return stmt, err
@ -64,7 +64,6 @@ func (p *Parser) ParseStatement() (stmt Statement, err error) {
return stmt, nil
}
/*
// parseExplain parses EXPLAIN [QUERY PLAN] STMT.
func (p *Parser) parseExplainStatement() (_ *ExplainStatement, err error) {
var tok Token
@ -89,7 +88,7 @@ func (p *Parser) parseExplainStatement() (_ *ExplainStatement, err error) {
return &stmt, err
}
return &stmt, nil
}*/
}
// parseStmt parses all statement types.
func (p *Parser) parseNonExplainStatement() (Statement, error) {
@ -102,10 +101,14 @@ func (p *Parser) parseNonExplainStatement() (Statement, error) {
return p.parseBulkInsertStatement()
case CREATE:
return p.parseCreateStatement()
case COPY:
return p.parseCopyStatement()
case DROP:
return p.parseDropStatement()
case SELECT:
return p.parseSelectStatement(false, nil)
case PREDICT:
return p.parsePredictStatement()
case INSERT, REPLACE:
return p.parseInsertStatement(nil)
case UPDATE:
@ -337,8 +340,10 @@ func (p *Parser) parseCreateStatement() (Statement, error) {
return p.parseCreateIndexStatement(pos)*/
case FUNCTION:
return p.parseCreateFunctionStatement(pos)
case MODEL:
return p.parseCreateModelStatement(pos)
default:
return nil, p.errorExpected(pos, tok, "DATABASE, TABLE, VIEW or FUNCTION")
return nil, p.errorExpected(pos, tok, "DATABASE, TABLE, VIEW, FUNCTION or MODEL")
}
}
@ -373,6 +378,8 @@ func (p *Parser) parseDropStatement() (Statement, error) {
return p.parseDropIndexStatement(pos)*/
case FUNCTION:
return p.parseDropFunctionStatement(pos)
case MODEL:
return p.parseDropModelStatement(pos)
default:
return nil, p.errorExpected(pos, tok, "DATABASE, TABLE, VIEW or FUNCTION")
}
@ -1158,6 +1165,72 @@ func (p *Parser) parseDropTableStatement(dropPos Pos) (_ *DropTableStatement, er
return &stmt, nil
}
func (p *Parser) parseCopyStatement() (_ *CopyStatement, err error) {
assert(p.peek() == COPY)
var stmt CopyStatement
stmt.Copy, _, _ = p.scan()
ident, err := p.parseIdent("table name")
if err != nil {
return &stmt, err
}
if stmt.Source, err = p.parseQualifiedTableName(ident); err != nil {
return &stmt, err
}
if p.peek() != TO {
return &stmt, p.errorExpected(p.pos, p.tok, "TO")
}
stmt.To, _, _ = p.scan()
if stmt.TargetName, err = p.parseIdent("table name"); err != nil {
return &stmt, err
}
// parse optional "WHERE expr"
if p.peek() == WHERE {
stmt.Where, _, _ = p.scan()
if stmt.WhereExpr, err = p.ParseExpr(); err != nil {
return &stmt, err
}
}
// options
if p.peek() == WITH {
stmt.With, _, _ = p.scan()
if !isCopyOptionStartToken(p.peek(), p) {
return &stmt, p.errorExpected(p.pos, p.tok, "URL or APIKEY")
}
for {
option, err := p.parseIdent("copy option")
if err != nil {
return &stmt, err
}
switch strings.ToLower(option.Name) {
case "url":
stmt.Url, err = p.ParseExpr()
if err != nil {
return &stmt, err
}
case "apikey":
stmt.ApiKey, err = p.ParseExpr()
if err != nil {
return &stmt, err
}
}
if !isCopyOptionStartToken(p.peek(), p) {
break
}
}
}
return &stmt, nil
}
func (p *Parser) parseCreateViewStatement(createPos Pos) (_ *CreateViewStatement, err error) {
assert(p.peek() == VIEW)
@ -1286,6 +1359,29 @@ func (p *Parser) parseDropViewStatement(dropPos Pos) (_ *DropViewStatement, err
return &stmt, nil
}
func (p *Parser) parseDropModelStatement(dropPos Pos) (_ *DropModelStatement, err error) {
assert(p.peek() == MODEL)
var stmt DropModelStatement
stmt.Drop = dropPos
stmt.Model, _, _ = p.scan()
// Parse optional "IF EXISTS".
if p.peek() == IF {
stmt.If, _, _ = p.scan()
if p.peek() != EXISTS {
return &stmt, p.errorExpected(p.pos, p.tok, "EXISTS")
}
stmt.IfExists, _, _ = p.scan()
}
if stmt.Name, err = p.parseIdent("view name"); err != nil {
return &stmt, err
}
return &stmt, nil
}
/*func (p *Parser) parseCreateIndexStatement(createPos Pos) (_ *CreateIndexStatement, err error) {
assert(p.peek() == INDEX || p.peek() == UNIQUE)
@ -1457,11 +1553,35 @@ func (p *Parser) parseCreateFunctionStatement(createPos Pos) (_ *CreateFunctionS
return &stmt, p.errorExpected(p.pos, p.tok, "RETURNS")
}
stmt.Returns, _, _ = p.scan()
stmt.ReturnDef, err = p.parseParameterDefinition()
stmt.ReturnType, err = p.parseType()
if err != nil {
return &stmt, err
}
// options
if p.peek() == WITH {
stmt.With, _, _ = p.scan()
stmt.Options = make([]*FunctionOptionDefinition, 0)
for {
option, err := p.parseIdent("function option")
if err != nil {
return &stmt, err
}
expr, err := p.ParseExpr()
if err != nil {
return &stmt, err
}
stmt.Options = append(stmt.Options, &FunctionOptionDefinition{
Name: option,
OptionExpr: expr,
})
if p.peek() == AS {
break
}
}
}
if p.peek() != AS {
return &stmt, p.errorExpected(p.pos, p.tok, "AS")
}
@ -1473,7 +1593,7 @@ func (p *Parser) parseCreateFunctionStatement(createPos Pos) (_ *CreateFunctionS
stmt.Begin, _, _ = p.scan()
for {
s, err := p.parseFunctionBodyStatement()
s, err := p.parseFunctionBodyStatement(&stmt)
if err != nil {
return &stmt, err
}
@ -1494,8 +1614,15 @@ func (p *Parser) parseCreateFunctionStatement(createPos Pos) (_ *CreateFunctionS
return &stmt, nil
}
func (p *Parser) parseFunctionBodyStatement() (stmt Statement, err error) {
func (p *Parser) parseFunctionBodyStatement(cf *CreateFunctionStatement) (stmt Statement, err error) {
switch p.peek() {
case RETURN:
s, err := p.parseReturnStatement()
if err != nil {
return stmt, err
}
cf.Body = append(cf.Body, s)
case END:
break
default:
@ -1507,6 +1634,20 @@ func (p *Parser) parseFunctionBodyStatement() (stmt Statement, err error) {
return stmt, nil
}
func (p *Parser) parseReturnStatement() (_ *ReturnStatement, err error) {
assert(p.peek() == RETURN)
var stmt ReturnStatement
stmt.Return, _, _ = p.scan()
expr, err := p.ParseExpr()
if err != nil {
return &stmt, err
}
stmt.ReturnExpr = expr
return &stmt, nil
}
func (p *Parser) parseDropFunctionStatement(dropPos Pos) (_ *DropFunctionStatement, err error) {
assert(p.peek() == FUNCTION)
@ -1530,6 +1671,69 @@ func (p *Parser) parseDropFunctionStatement(dropPos Pos) (_ *DropFunctionStateme
return &stmt, nil
}
func (p *Parser) parseCreateModelStatement(createPos Pos) (_ *CreateModelStatement, err error) {
assert(p.peek() == MODEL)
var stmt CreateModelStatement
stmt.Create = createPos
stmt.Model, _, _ = p.scan()
// Parse optional "IF NOT EXISTS".
if p.peek() == IF {
stmt.If, _, _ = p.scan()
if p.peek() != NOT {
return &stmt, p.errorExpected(p.pos, p.tok, "NOT")
}
stmt.IfNot, _, _ = p.scan()
if p.peek() != EXISTS {
return &stmt, p.errorExpected(p.pos, p.tok, "EXISTS")
}
stmt.IfNotExists, _, _ = p.scan()
}
if stmt.Name, err = p.parseIdent("model name"); err != nil {
return &stmt, err
}
// options
if p.peek() != WITH {
return &stmt, p.errorExpected(p.pos, p.tok, "WITH")
}
stmt.With, _, _ = p.scan()
stmt.Options = make([]*ModelOptionDefinition, 0)
for {
option, err := p.parseIdent("model option")
if err != nil {
return &stmt, err
}
expr, err := p.ParseExpr()
if err != nil {
return &stmt, err
}
stmt.Options = append(stmt.Options, &ModelOptionDefinition{
Name: option,
OptionExpr: expr,
})
if p.peek() == AS {
break
}
}
if p.peek() != AS {
return &stmt, p.errorExpected(p.pos, p.tok, "AS")
}
stmt.As, _, _ = p.scan()
if stmt.ModelQuery, err = p.parseSelectStatement(false, nil); err != nil {
return &stmt, err
}
return &stmt, nil
}
func (p *Parser) parseIdent(desc string) (*Ident, error) {
pos, tok, lit := p.scan()
switch tok {
@ -2152,82 +2356,87 @@ func (p *Parser) parseSelectStatement(compounded bool, withClause *WithClause) (
// }
//}
switch p.peek() {
/*case VALUES:
stmt.Values, _, _ = p.scan()
if p.peek() != SELECT {
return &stmt, p.errorExpected(p.pos, p.tok, "SELECT")
}
for {
var list ExprList
if p.peek() != LP {
return &stmt, p.errorExpected(p.pos, p.tok, "left paren")
stmt.Select, _, _ = p.scan()
// Parse optional "DISTINCT".
if tok := p.peek(); tok == DISTINCT {
stmt.Distinct, _, _ = p.scan()
}
if p.peek() == TOP {
stmt.Top, _, _ = p.scan()
if p.peek() == LP {
_, _, _ = p.scan()
}
list.Lparen, _, _ = p.scan()
if stmt.TopExpr, err = p.ParseExpr(); err != nil {
return &stmt, err
}
if p.peek() == RP {
_, _, _ = p.scan()
}
}
if p.peek() == TOPN {
stmt.TopN, _, _ = p.scan()
if p.peek() == LP {
_, _, _ = p.scan()
}
if stmt.TopExpr, err = p.ParseExpr(); err != nil {
return &stmt, err
}
if p.peek() == RP {
_, _, _ = p.scan()
}
}
// Parse result columns.
for {
col, err := p.parseResultColumn()
if err != nil {
return &stmt, err
}
stmt.Columns = append(stmt.Columns, col)
if p.peek() != COMMA {
break
}
p.scan()
}
// Parse FROM clause.
if p.peek() == FROM {
stmt.From, _, _ = p.scan()
if stmt.Source, err = p.parseSource(); err != nil {
return &stmt, err
}
}
// Parse WHERE clause.
if p.peek() == WHERE {
stmt.Where, _, _ = p.scan()
if stmt.WhereExpr, err = p.ParseExpr(); err != nil {
return &stmt, err
}
}
// Parse GROUP BY/HAVING clause.
if p.peek() == GROUP {
stmt.Group, _, _ = p.scan()
if p.peek() != BY {
return &stmt, p.errorExpected(p.pos, p.tok, "BY")
}
stmt.GroupBy, _, _ = p.scan()
for {
expr, err := p.ParseExpr()
if err != nil {
return &stmt, err
}
list.Exprs = append(list.Exprs, expr)
if p.peek() == RP {
break
} else if p.peek() != COMMA {
return &stmt, p.errorExpected(p.pos, p.tok, "comma or right paren")
}
p.scan()
}
list.Rparen, _, _ = p.scan()
stmt.ValueLists = append(stmt.ValueLists, &list)
if p.peek() != COMMA {
break
}
p.scan()
}*/
case SELECT:
stmt.Select, _, _ = p.scan()
// Parse optional "DISTINCT".
if tok := p.peek(); tok == DISTINCT {
stmt.Distinct, _, _ = p.scan()
}
if p.peek() == TOP {
stmt.Top, _, _ = p.scan()
if p.peek() == LP {
_, _, _ = p.scan()
}
if stmt.TopExpr, err = p.ParseExpr(); err != nil {
return &stmt, err
}
if p.peek() == RP {
_, _, _ = p.scan()
}
}
if p.peek() == TOPN {
stmt.TopN, _, _ = p.scan()
if p.peek() == LP {
_, _, _ = p.scan()
}
if stmt.TopExpr, err = p.ParseExpr(); err != nil {
return &stmt, err
}
if p.peek() == RP {
_, _, _ = p.scan()
}
}
// Parse result columns.
for {
col, err := p.parseResultColumn()
if err != nil {
return &stmt, err
}
stmt.Columns = append(stmt.Columns, col)
stmt.GroupByExprs = append(stmt.GroupByExprs, expr)
if p.peek() != COMMA {
break
@ -2235,101 +2444,61 @@ func (p *Parser) parseSelectStatement(compounded bool, withClause *WithClause) (
p.scan()
}
// Parse FROM clause.
if p.peek() == FROM {
stmt.From, _, _ = p.scan()
if stmt.Source, err = p.parseSource(); err != nil {
// Parse optional HAVING clause.
if p.peek() == HAVING {
stmt.Having, _, _ = p.scan()
if stmt.HavingExpr, err = p.ParseExpr(); err != nil {
return &stmt, err
}
}
// Parse WHERE clause.
if p.peek() == WHERE {
stmt.Where, _, _ = p.scan()
if stmt.WhereExpr, err = p.ParseExpr(); err != nil {
return &stmt, err
}
}
// Parse GROUP BY/HAVING clause.
if p.peek() == GROUP {
stmt.Group, _, _ = p.scan()
if p.peek() != BY {
return &stmt, p.errorExpected(p.pos, p.tok, "BY")
}
stmt.GroupBy, _, _ = p.scan()
for {
expr, err := p.ParseExpr()
if err != nil {
return &stmt, err
}
stmt.GroupByExprs = append(stmt.GroupByExprs, expr)
if p.peek() != COMMA {
break
}
p.scan()
}
// Parse optional HAVING clause.
if p.peek() == HAVING {
stmt.Having, _, _ = p.scan()
if stmt.HavingExpr, err = p.ParseExpr(); err != nil {
return &stmt, err
}
}
}
// Parse WINDOW clause.
if p.peek() == WINDOW {
stmt.Window, _, _ = p.scan()
for {
var window Window
if window.Name, err = p.parseIdent("window name"); err != nil {
return &stmt, err
}
if p.peek() != AS {
return &stmt, p.errorExpected(p.pos, p.tok, "AS")
}
window.As, _, _ = p.scan()
if window.Definition, err = p.parseWindowDefinition(); err != nil {
return &stmt, err
}
stmt.Windows = append(stmt.Windows, &window)
if p.peek() != COMMA {
break
}
p.scan()
}
}
default:
return &stmt, p.errorExpected(p.pos, p.tok, "SELECT")
}
// Parse WINDOW clause.
// if p.peek() == WINDOW {
// stmt.Window, _, _ = p.scan()
// for {
// var window Window
// if window.Name, err = p.parseIdent("window name"); err != nil {
// return &stmt, err
// }
// if p.peek() != AS {
// return &stmt, p.errorExpected(p.pos, p.tok, "AS")
// }
// window.As, _, _ = p.scan()
// if window.Definition, err = p.parseWindowDefinition(); err != nil {
// return &stmt, err
// }
// stmt.Windows = append(stmt.Windows, &window)
// if p.peek() != COMMA {
// break
// }
// p.scan()
// }
// }
// Optionally compound additional SELECT/VALUES.
switch tok := p.peek(); tok {
case UNION, INTERSECT, EXCEPT:
if tok == UNION {
stmt.Union, _, _ = p.scan()
if p.peek() == ALL {
stmt.UnionAll, _, _ = p.scan()
}
} else if tok == INTERSECT {
stmt.Intersect, _, _ = p.scan()
} else {
stmt.Except, _, _ = p.scan()
}
// switch tok := p.peek(); tok {
// case UNION, INTERSECT, EXCEPT:
// if tok == UNION {
// stmt.Union, _, _ = p.scan()
// if p.peek() == ALL {
// stmt.UnionAll, _, _ = p.scan()
// }
// } else if tok == INTERSECT {
// stmt.Intersect, _, _ = p.scan()
// } else {
// stmt.Except, _, _ = p.scan()
// }
if stmt.Compound, err = p.parseSelectStatement(true, nil); err != nil {
return &stmt, err
}
}
// if stmt.Compound, err = p.parseSelectStatement(true, nil); err != nil {
// return &stmt, err
// }
// }
// Parse ORDER BY clause.
if !compounded && p.peek() == ORDER {
@ -2353,6 +2522,13 @@ func (p *Parser) parseSelectStatement(compounded bool, withClause *WithClause) (
}
}
// Parse LIMIT clause.
if !compounded && p.peek() == LIMIT {
stmt.Limit, _, _ = p.scan()
if stmt.LimitExpr, err = p.ParseExpr(); err != nil {
return &stmt, err
}
}
return &stmt, nil
}
@ -2704,6 +2880,26 @@ func (p *Parser) parseTableValuedFunction(ident *Ident) (_ *TableValuedFunction,
return &cte, nil
}*/
func (p *Parser) parsePredictStatement() (_ *PredictStatement, err error) {
assert(p.peek() == PREDICT)
var stmt PredictStatement
stmt.Predict, _, _ = p.scan()
if p.peek() != USING {
return &stmt, p.errorExpected(p.pos, p.tok, "USING")
}
stmt.Using, _, _ = p.scan()
if stmt.ModelName, err = p.parseIdent("model name"); err != nil {
return &stmt, err
}
if stmt.InputQuery, err = p.parseSelectStatement(false, nil); err != nil {
return &stmt, err
}
return &stmt, nil
}
func (p *Parser) mustParseLiteral() Expr {
assert(isLiteralToken(p.tok))
pos, tok, lit := p.scan()
@ -2748,7 +2944,7 @@ func (p *Parser) parseOperand() (expr Expr, err error) {
return p.parseCall(ident)
}
return nil, p.errorExpected(p.pos, pk, "call expression")
case STRING:
case STRING, BLOB:
return &StringLit{ValuePos: pos, Value: lit}, nil
case FLOAT:
return &FloatLit{ValuePos: pos, Value: lit}, nil
@ -3633,6 +3829,22 @@ func isBulkInsertOptionStartToken(tok Token, p *Parser) bool {
return false
}
func isCopyOptionStartToken(tok Token, p *Parser) bool {
switch tok {
case IDENT:
ident, err := p.parseIdent("copy option")
defer p.unscan()
if err != nil {
return false
}
switch strings.ToUpper(ident.Name) {
case "URL", "APIKEY":
return true
}
}
return false
}
// isConstraintStartToken returns true if tok is the initial token of a constraint.
func isConstraintStartToken(tok Token, isTable bool) bool {
switch tok {

191
sql3/parser/parser_test.go
View file

@ -472,7 +472,7 @@ func TestParser_ParseAlterStatement(t *testing.T) {
func TestParser_ParseFunctionStatement(t *testing.T) {
t.Run("CreateFunction", func(t *testing.T) {
AssertParseStatement(t, `CREATE FUNCTION IF NOT EXISTS func (@param1 int, @param2 string) returns @scalar int as begin end`, &parser.CreateFunctionStatement{
AssertParseStatement(t, `CREATE FUNCTION IF NOT EXISTS func (@param1 int, @param2 string) returns int as begin end`, &parser.CreateFunctionStatement{
Create: pos(0),
Function: pos(7),
If: pos(16),
@ -490,15 +490,12 @@ func TestParser_ParseFunctionStatement(t *testing.T) {
Type: &parser.Type{Name: &parser.Ident{NamePos: pos(57), Name: "string"}},
},
},
Rparen: pos(63),
Returns: pos(65),
ReturnDef: &parser.ParameterDefinition{
Name: &parser.Variable{Name: "@scalar", NamePos: pos(73)},
Type: &parser.Type{Name: &parser.Ident{NamePos: pos(81), Name: "int"}},
},
As: pos(85),
Begin: pos(88),
End: pos(94),
Rparen: pos(63),
Returns: pos(65),
ReturnType: &parser.Type{Name: &parser.Ident{NamePos: pos(73), Name: "int"}},
As: pos(77),
Begin: pos(80),
End: pos(86),
})
// AssertParseStatement(t, `CREATE TRIGGER IF NOT EXISTS trig BEFORE INSERT ON tbl BEGIN DELETE FROM new; END`, &parser.CreateFunctionStatement{
// Create: pos(0),
@ -955,7 +952,7 @@ func TestParser_ParseStatement(t *testing.T) {
},
})
AssertParseStatementError(t, `CREATE`, `1:1: expected DATABASE, TABLE, VIEW or FUNCTION`)
AssertParseStatementError(t, `CREATE`, `1:1: expected DATABASE, TABLE, VIEW, FUNCTION or MODEL`)
AssertParseStatementError(t, `CREATE DATABASE`, `1:15: expected database name, found 'EOF'`)
AssertParseStatementError(t, `CREATE DATABASE IF`, `1:18: expected NOT, found 'EOF'`)
AssertParseStatementError(t, `CREATE DATABASE IF NOT`, `1:22: expected EXISTS, found 'EOF'`)
@ -2215,29 +2212,29 @@ func TestParser_ParseStatement(t *testing.T) {
Having: pos(22),
HavingExpr: &parser.BoolLit{ValuePos: pos(29), Value: true},
})
AssertParseStatement(t, `SELECT * WINDOW win1 AS (), win2 AS ()`, &parser.SelectStatement{
Select: pos(0),
Columns: []*parser.ResultColumn{{Star: pos(7)}},
Window: pos(9),
Windows: []*parser.Window{
{
Name: &parser.Ident{NamePos: pos(16), Name: "win1"},
As: pos(21),
Definition: &parser.WindowDefinition{
Lparen: pos(24),
Rparen: pos(25),
},
},
{
Name: &parser.Ident{NamePos: pos(28), Name: "win2"},
As: pos(33),
Definition: &parser.WindowDefinition{
Lparen: pos(36),
Rparen: pos(37),
},
},
},
})
// AssertParseStatement(t, `SELECT * WINDOW win1 AS (), win2 AS ()`, &parser.SelectStatement{
// Select: pos(0),
// Columns: []*parser.ResultColumn{{Star: pos(7)}},
// Window: pos(9),
// Windows: []*parser.Window{
// {
// Name: &parser.Ident{NamePos: pos(16), Name: "win1"},
// As: pos(21),
// Definition: &parser.WindowDefinition{
// Lparen: pos(24),
// Rparen: pos(25),
// },
// },
// {
// Name: &parser.Ident{NamePos: pos(28), Name: "win2"},
// As: pos(33),
// Definition: &parser.WindowDefinition{
// Lparen: pos(36),
// Rparen: pos(37),
// },
// },
// },
// })
AssertParseStatement(t, `SELECT * ORDER BY foo ASC, bar DESC`, &parser.SelectStatement{
Select: pos(0),
@ -2252,64 +2249,64 @@ func TestParser_ParseStatement(t *testing.T) {
},
})
AssertParseStatement(t, `SELECT * UNION SELECT * ORDER BY foo`, &parser.SelectStatement{
Select: pos(0),
Columns: []*parser.ResultColumn{
{Star: pos(7)},
},
Union: pos(9),
Compound: &parser.SelectStatement{
Select: pos(15),
Columns: []*parser.ResultColumn{
{Star: pos(22)},
},
},
Order: pos(24),
OrderBy: pos(30),
OrderingTerms: []*parser.OrderingTerm{
{X: &parser.Ident{NamePos: pos(33), Name: "foo"}},
},
})
AssertParseStatement(t, `SELECT * UNION ALL SELECT *`, &parser.SelectStatement{
Select: pos(0),
Columns: []*parser.ResultColumn{
{Star: pos(7)},
},
Union: pos(9),
UnionAll: pos(15),
Compound: &parser.SelectStatement{
Select: pos(19),
Columns: []*parser.ResultColumn{
{Star: pos(26)},
},
},
})
AssertParseStatement(t, `SELECT * INTERSECT SELECT *`, &parser.SelectStatement{
Select: pos(0),
Columns: []*parser.ResultColumn{
{Star: pos(7)},
},
Intersect: pos(9),
Compound: &parser.SelectStatement{
Select: pos(19),
Columns: []*parser.ResultColumn{
{Star: pos(26)},
},
},
})
AssertParseStatement(t, `SELECT * EXCEPT SELECT *`, &parser.SelectStatement{
Select: pos(0),
Columns: []*parser.ResultColumn{
{Star: pos(7)},
},
Except: pos(9),
Compound: &parser.SelectStatement{
Select: pos(16),
Columns: []*parser.ResultColumn{
{Star: pos(23)},
},
},
})
// AssertParseStatement(t, `SELECT * UNION SELECT * ORDER BY foo`, &parser.SelectStatement{
// Select: pos(0),
// Columns: []*parser.ResultColumn{
// {Star: pos(7)},
// },
// Union: pos(9),
// Compound: &parser.SelectStatement{
// Select: pos(15),
// Columns: []*parser.ResultColumn{
// {Star: pos(22)},
// },
// },
// Order: pos(24),
// OrderBy: pos(30),
// OrderingTerms: []*parser.OrderingTerm{
// {X: &parser.Ident{NamePos: pos(33), Name: "foo"}},
// },
// })
// AssertParseStatement(t, `SELECT * UNION ALL SELECT *`, &parser.SelectStatement{
// Select: pos(0),
// Columns: []*parser.ResultColumn{
// {Star: pos(7)},
// },
// Union: pos(9),
// UnionAll: pos(15),
// Compound: &parser.SelectStatement{
// Select: pos(19),
// Columns: []*parser.ResultColumn{
// {Star: pos(26)},
// },
// },
// })
// AssertParseStatement(t, `SELECT * INTERSECT SELECT *`, &parser.SelectStatement{
// Select: pos(0),
// Columns: []*parser.ResultColumn{
// {Star: pos(7)},
// },
// Intersect: pos(9),
// Compound: &parser.SelectStatement{
// Select: pos(19),
// Columns: []*parser.ResultColumn{
// {Star: pos(26)},
// },
// },
// })
// AssertParseStatement(t, `SELECT * EXCEPT SELECT *`, &parser.SelectStatement{
// Select: pos(0),
// Columns: []*parser.ResultColumn{
// {Star: pos(7)},
// },
// Except: pos(9),
// Compound: &parser.SelectStatement{
// Select: pos(16),
// Columns: []*parser.ResultColumn{
// {Star: pos(23)},
// },
// },
// })
/*AssertParseStatement(t, `VALUES (1, 2), (3, 4)`, &parser.SelectStatement{
Values: pos(0),
@ -3402,11 +3399,11 @@ func TestParser_ParseStatement(t *testing.T) {
AssertParseStatementError(t, `SELECT * GROUP BY`, `1:17: expected expression, found 'EOF'`)
AssertParseStatementError(t, `SELECT * GROUP BY foo bar`, `1:23: expected semicolon or EOF, found bar`)
AssertParseStatementError(t, `SELECT * GROUP BY foo HAVING`, `1:28: expected expression, found 'EOF'`)
AssertParseStatementError(t, `SELECT * WINDOW`, `1:15: expected window name, found 'EOF'`)
AssertParseStatementError(t, `SELECT * WINDOW win1`, `1:20: expected AS, found 'EOF'`)
AssertParseStatementError(t, `SELECT * WINDOW win1 AS`, `1:23: expected left paren, found 'EOF'`)
AssertParseStatementError(t, `SELECT * WINDOW win1 AS (`, `1:25: expected right paren, found 'EOF'`)
AssertParseStatementError(t, `SELECT * WINDOW win1 AS () win2`, `1:28: expected semicolon or EOF, found win2`)
// AssertParseStatementError(t, `SELECT * WINDOW`, `1:15: expected window name, found 'EOF'`)
// AssertParseStatementError(t, `SELECT * WINDOW win1`, `1:20: expected AS, found 'EOF'`)
// AssertParseStatementError(t, `SELECT * WINDOW win1 AS`, `1:23: expected left paren, found 'EOF'`)
// AssertParseStatementError(t, `SELECT * WINDOW win1 AS (`, `1:25: expected right paren, found 'EOF'`)
// AssertParseStatementError(t, `SELECT * WINDOW win1 AS () win2`, `1:28: expected semicolon or EOF, found win2`)
AssertParseStatementError(t, `SELECT * ORDER`, `1:14: expected BY, found 'EOF'`)
AssertParseStatementError(t, `SELECT * ORDER BY`, `1:17: expected expression, found 'EOF'`)
AssertParseStatementError(t, `SELECT * ORDER BY 1,`, `1:20: expected expression, found 'EOF'`)

14
sql3/parser/token.go
View file

@ -78,8 +78,6 @@ const (
ACTION
ADD
AFTER
AGG_COLUMN
AGG_FUNCTION
ALL
ALTER
ANALYZE
@ -106,9 +104,9 @@ const (
COMMENT
CONFLICT
CONSTRAINT
COPY
CREATE
CROSS
CTIME_KW
CURRENT
CURRENT_DATE
CURRENT_TIMESTAMP
@ -167,11 +165,13 @@ const (
LAST
LEFT
LIKE
LIMIT
LRU
MAP
MATCH
MAX
MIN
MODEL
NO
NOT
NOTBETWEEN
@ -194,6 +194,7 @@ const (
PLAN
PRAGMA
PRECEDING
PREDICT
PRIMARY
QUERY
RANGE
@ -304,8 +305,6 @@ var tokens = [...]string{
ACTION: "ACTION",
ADD: "ADD",
AFTER: "AFTER",
AGG_COLUMN: "AGG_COLUMN",
AGG_FUNCTION: "AGG_FUNCTION",
ALL: "ALL",
ALTER: "ALTER",
ANALYZE: "ANALYZE",
@ -332,9 +331,9 @@ var tokens = [...]string{
COMMENT: "COMMENT",
CONFLICT: "CONFLICT",
CONSTRAINT: "CONSTRAINT",
COPY: "COPY",
CREATE: "CREATE",
CROSS: "CROSS",
CTIME_KW: "CTIME_KW",
CURRENT: "CURRENT",
CURRENT_DATE: "CURRENT_DATE",
CURRENT_TIMESTAMP: "CURRENT_TIMESTAMP",
@ -393,11 +392,13 @@ var tokens = [...]string{
LAST: "LAST",
LEFT: "LEFT",
LIKE: "LIKE",
LIMIT: "LIMIT",
MAP: "MAP",
LRU: "LRU",
MATCH: "MATCH",
MAX: "MAX",
MIN: "MIN",
MODEL: "MODEL",
NO: "NO",
NOT: "NOT",
NOTBETWEEN: "NOTBETWEEN",
@ -420,6 +421,7 @@ var tokens = [...]string{
PLAN: "PLAN",
PRAGMA: "PRAGMA",
PRECEDING: "PRECEDING",
PREDICT: "PREDICT",
PRIMARY: "PRIMARY",
QUERY: "QUERY",
RANGE: "RANGE",

120
sql3/planner/compilecopy.go Normal file
View file

@ -0,0 +1,120 @@
// Copyright 2022 Molecula Corp. All rights reserved.
package planner
import (
"context"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
// compileCopyStatement compiles a parser.CopyStatement AST into a PlanOperator
func (p *ExecutionPlanner) compileCopyStatement(stmt *parser.CopyStatement) (types.PlanOperator, error) {
query := NewPlanOpQuery(p, NewPlanOpNullTable(), p.sql)
query.AddWarning("🦖 here there be dragons! COPY statement is experimental.")
// handle projections
projections := make([]types.PlanExpression, 0)
for _, c := range stmt.Source.PossibleOutputColumns() {
expr := &parser.QualifiedRef{
Table: &parser.Ident{Name: c.TableName},
Column: &parser.Ident{Name: c.ColumnName},
ColumnIndex: c.ColumnIndex,
RefDataType: c.Datatype,
}
planExpr, err := p.compileExpr(expr)
if err != nil {
return nil, err
}
projections = append(projections, planExpr)
}
// handle the where clause
where, err := p.compileExpr(stmt.WhereExpr)
if err != nil {
return nil, err
}
// compile source
source, err := p.compileSource(query, stmt.Source)
if err != nil {
return nil, err
}
// if we did have a where, insert the filter op
if where != nil {
source = NewPlanOpFilter(p, where, source)
}
var compiledOp types.PlanOperator
url := ""
apiKey := ""
if stmt.Url != nil {
lit, ok := stmt.Url.(*parser.StringLit)
if !ok {
return nil, sql3.NewErrStringLiteral(stmt.Url.Pos().Line, stmt.Url.Pos().Column)
}
url = lit.Value
}
if stmt.ApiKey != nil {
lit, ok := stmt.ApiKey.(*parser.StringLit)
if !ok {
return nil, sql3.NewErrStringLiteral(stmt.ApiKey.Pos().Line, stmt.ApiKey.Pos().Column)
}
apiKey = lit.Value
}
// get the source table
tname := dax.TableName(stmt.Source.String())
tbl, err := p.schemaAPI.TableByName(context.Background(), tname)
if err != nil {
if isTableNotFoundError(err) {
return nil, sql3.NewErrTableNotFound(0, 0, stmt.Source.String())
}
return nil, err
}
// get the ddl of source table and subst target table name
ddl := generateTableDDL(tbl, stmt.TargetName.Name)
compiledOp = NewPlanOpCopy(p, stmt.TargetName.Name, url, apiKey, ddl, NewPlanOpProjection(projections, source))
children := []types.PlanOperator{
compiledOp,
}
return query.WithChildren(children...)
}
func (p *ExecutionPlanner) analyzeCopyStatement(ctx context.Context, stmt *parser.CopyStatement) error {
// analyze source
var err error
source, err := p.analyzeSource(ctx, stmt.Source, stmt)
if err != nil {
return err
}
stmt.Source = source
// analyze where
expr, err := p.analyzeExpression(ctx, stmt.WhereExpr, stmt)
if err != nil {
return err
}
stmt.WhereExpr = expr
expr, err = p.analyzeExpression(ctx, stmt.Url, stmt)
if err != nil {
return err
}
stmt.Url = expr
expr, err = p.analyzeExpression(ctx, stmt.ApiKey, stmt)
if err != nil {
return err
}
stmt.ApiKey = expr
return nil
}

79
sql3/planner/compilecreatefunction.go Normal file
View file

@ -0,0 +1,79 @@
// Copyright 2023 Molecula Corp. All rights reserved.
package planner
import (
"strings"
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
// compileCreateFunctionStatement compiles a parser.CreateFunctionStatement AST into a PlanOperator
func (p *ExecutionPlanner) compileCreateFunctionStatement(stmt *parser.CreateFunctionStatement) (types.PlanOperator, error) {
functionName := parser.IdentName(stmt.Name)
function := &functionSystemObject{
name: functionName,
}
lang := "sql"
if len(stmt.Options) > 0 {
for _, o := range stmt.Options {
switch strings.ToLower(o.Name.String()) {
case "language":
lit, ok := o.OptionExpr.(*parser.StringLit)
if !ok {
return nil, sql3.NewErrStringLiteral(o.OptionExpr.Pos().Line, o.OptionExpr.Pos().Column)
}
l := strings.ToLower(lit.Value)
switch l {
case "python":
lang = l
default:
return nil, sql3.NewErrInternalf("unsupported language '%s'", l)
}
}
}
}
function.language = lang
// TODO(pok) - hobble user defined functions for now
switch lang {
case "sql":
return nil, sql3.NewErrInternalf("unsupported language '%s'", lang)
case "python":
// return nil, sql3.NewErrInternalf("unsupported language '%s'", lang)
// function body is in the return statement
if len(stmt.Body) != 1 {
return nil, sql3.NewErrInternalf("unexpected body len '%d'", len(stmt.Body))
}
rs, ok := stmt.Body[0].(*parser.ReturnStatement)
if !ok {
return nil, sql3.NewErrInternalf("unexpected statement type '%T'", stmt.Body[0])
}
bexpr, ok := rs.ReturnExpr.(*parser.StringLit)
if !ok {
return nil, sql3.NewErrInternalf("unexpected expression type '%T'", rs.ReturnExpr)
}
function.body = bexpr.Value
default:
return nil, sql3.NewErrInternalf("unsupported language '%s'", lang)
}
fn := NewPlanOpCreateFunction(p, stmt.IfNotExists.IsValid(), function)
fn.AddWarning("🦖 here there be dragons! CREATE FUNCTION statement is experimental.")
query := NewPlanOpQuery(p, fn, p.sql)
return query, nil
}
func (p *ExecutionPlanner) analyzeCreateFunctionStatement(stmt *parser.CreateFunctionStatement) error {
return nil
}

180
sql3/planner/compilecreatemodel.go Normal file
View file

@ -0,0 +1,180 @@
// Copyright 2022 Molecula Corp. All rights reserved.
package planner
import (
"context"
"strings"
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
// TODO (pok) what does 'if not exists' do?
// compileCreateModelStatement compiles a parser.CreateModelStatement AST into a PlanOperator
func (p *ExecutionPlanner) compileCreateModelStatement(stmt *parser.CreateModelStatement) (types.PlanOperator, error) {
modelName := parser.IdentName(stmt.Name)
// does the model exist
obj, err := p.getModelByName(modelName)
if err != nil {
return nil, err
}
if obj != nil {
return nil, sql3.NewErrInternalf("model '%s' already exists", modelName)
}
// if we got to here model does not exist
model := &modelSystemObject{
name: modelName,
}
for _, o := range stmt.Options {
optName := parser.IdentName(o.Name)
switch strings.ToLower(optName) {
case "modeltype":
lit, ok := o.OptionExpr.(*parser.StringLit)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", o.OptionExpr)
}
model.modelType = lit.Value
case "labels":
lit, ok := o.OptionExpr.(*parser.SetLiteralExpr)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", o.OptionExpr)
}
model.labels = make([]string, len(lit.Members))
for i, m := range lit.Members {
mlit, ok := m.(*parser.StringLit)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", m)
}
model.labels[i] = mlit.Value
}
default:
return nil, sql3.NewErrInternalf("unexpected model option '%s'", optName)
}
}
selOp, err := p.compileSelectStatement(stmt.ModelQuery, true)
if err != nil {
return nil, err
}
// build a list of input columns for the model from the select query
schema := selOp.Schema()
model.inputColumns = make([]string, 0)
for _, p := range schema {
// if we have no column name, we have an error
if len(p.ColumnName) == 0 {
return nil, sql3.NewErrInternalf("query output columns used as inputs to models must be named")
}
// exclude any that are in the labels
isLabel := false
for _, l := range model.labels {
if strings.EqualFold(p.ColumnName, l) {
isLabel = true
break
}
}
if !isLabel {
model.inputColumns = append(model.inputColumns, p.ColumnName)
}
}
createModel := NewPlanOpCreateModel(p, model, selOp)
createModel.AddWarning("🦖 here there be dragons! CREATE MODEL statement is experimental.")
query := NewPlanOpQuery(p, createModel, p.sql)
return query, nil
}
func (p *ExecutionPlanner) analyzeCreateModelStatement(ctx context.Context, stmt *parser.CreateModelStatement) error {
// iterate the options
for _, opt := range stmt.Options {
optName := parser.IdentName(opt.Name)
if !isValidModelOption(optName) {
return sql3.NewErrInternalf("invalid model option '%s'", optName)
}
e, err := p.analyzeModelOptionExpr(ctx, optName, opt.OptionExpr, stmt)
if err != nil {
return err
}
opt.OptionExpr = e
}
// analyze the select
_, err := p.analyzeSelectStatement(ctx, stmt.ModelQuery)
if err != nil {
return err
}
return nil
}
func isValidModelOption(name string) bool {
switch strings.ToLower(name) {
case "modeltype":
return true
case "labels":
return true
default:
return false
}
}
func (p *ExecutionPlanner) analyzeModelOptionExpr(ctx context.Context, optName string, expr parser.Expr, scope parser.Statement) (parser.Expr, error) {
if expr == nil {
return nil, nil
}
e, err := p.analyzeExpression(ctx, expr, scope)
if err != nil {
return nil, err
}
switch strings.ToLower(optName) {
case "modeltype":
// model type needs to be a string literal
if !(e.IsLiteral() && typeIsString(e.DataType())) {
return nil, sql3.NewErrStringLiteral(e.Pos().Line, e.Pos().Column)
}
ty, ok := e.(*parser.StringLit)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", e)
}
// these are the model types supported
switch strings.ToLower(ty.Value) {
case "linear_regresssion":
break
default:
return nil, sql3.NewErrInternalf("unexpected model tyoe '%s'", ty.Value)
}
return e, nil
case "labels":
// labels needs to be a string array literal
// TODO (pok) revist 'set' literals (should be array literal; type checking could be robustified etc.)
if !e.IsLiteral() {
return nil, sql3.NewErrInternalf("string array literal expected")
}
ok, baseType := typeIsSet(e.DataType())
if !ok {
return nil, sql3.NewErrInternalf("array expression expected")
}
if !typeIsString(baseType) {
return nil, sql3.NewErrInternalf("string array expected")
}
return e, nil
default:
return nil, sql3.NewErrInternalf("unexpected option name '%s'", optName)
}
}

26
sql3/planner/compiledropmodel.go Normal file
View file

@ -0,0 +1,26 @@
// Copyright 2023 Molecula Corp. All rights reserved.
package planner
import (
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
// compileDropModelStatement compiles a DROP MODEL statement into a PlanOperator.
func (p *ExecutionPlanner) compileDropModelStatement(stmt *parser.DropModelStatement) (_ types.PlanOperator, err error) {
modelName := parser.IdentName(stmt.Name)
v, err := p.getModelByName(modelName)
if err != nil {
return nil, err
}
if v == nil && !stmt.IfExists.IsValid() {
return nil, sql3.NewErrModelNotFound(0, 0, modelName)
}
dropModel := NewPlanOpDropModel(p, stmt.IfExists.IsValid(), modelName)
dropModel.AddWarning("🦖 here there be dragons! DROP MODEL statement is experimental.")
return NewPlanOpQuery(p, dropModel, p.sql), nil
}

50
sql3/planner/compilepredict.go Normal file
View file

@ -0,0 +1,50 @@
// Copyright 2022 Molecula Corp. All rights reserved.
package planner
import (
"context"
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
// compilePredictStatement compiles a parser.PredictStatement AST into a PlanOperator
func (p *ExecutionPlanner) compilePredictStatement(ctx context.Context, stmt *parser.PredictStatement) (types.PlanOperator, error) {
// go get the model
modelName := parser.IdentName(stmt.ModelName)
// does the model exist
obj, err := p.getModelByName(modelName)
if err != nil {
return nil, err
}
if obj == nil {
return nil, sql3.NewErrInternalf("model '%s' not found", modelName)
}
selOp, err := p.compileSelectStatement(stmt.InputQuery, true)
if err != nil {
return nil, err
}
predict := NewPlanOpPredict(p, obj, selOp)
predict.AddWarning("🦖 here there be dragons! PREDICT statement is experimental.")
query := NewPlanOpQuery(p, predict, p.sql)
return query, nil
}
func (p *ExecutionPlanner) analyzePredictStatement(ctx context.Context, stmt *parser.PredictStatement) error {
// analyze the select
_, err := p.analyzeSelectStatement(ctx, stmt.InputQuery)
if err != nil {
return err
}
return nil
}

25
sql3/planner/compileselect.go
View file

@ -305,7 +305,7 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement,
}
}
// insert the top operator if it exists
// insert the top operator if it exists, or limit - analyzer should have caught the case of both existing
if stmt.Top.IsValid() {
topExpr, err := p.compileExpr(stmt.TopExpr)
if err != nil {
@ -313,6 +313,14 @@ func (p *ExecutionPlanner) compileSelectStatement(stmt *parser.SelectStatement,
}
compiledOp = NewPlanOpTop(topExpr, compiledOp)
}
// handle limit
if stmt.Limit.IsValid() {
limitExpr, err := p.compileExpr(stmt.LimitExpr)
if err != nil {
return nil, err
}
compiledOp = NewPlanOpTop(limitExpr, compiledOp)
}
// handle distinct
if stmt.Distinct.IsValid() {
@ -613,6 +621,10 @@ func (p *ExecutionPlanner) analyzeSelectStatement(ctx context.Context, stmt *par
}
}
if stmt.TopExpr != nil && stmt.LimitExpr != nil {
return nil, sql3.NewErrErrTopLimitCannotCoexist(stmt.TopExpr.Pos().Line, stmt.TopExpr.Pos().Column)
}
expr, err := p.analyzeExpression(ctx, stmt.TopExpr, stmt)
if err != nil {
return nil, err
@ -624,6 +636,17 @@ func (p *ExecutionPlanner) analyzeSelectStatement(ctx context.Context, stmt *par
stmt.TopExpr = expr
}
expr, err = p.analyzeExpression(ctx, stmt.LimitExpr, stmt)
if err != nil {
return nil, err
}
if expr != nil {
if !(expr.IsLiteral() && typeIsInteger(expr.DataType())) {
return nil, sql3.NewErrIntegerLiteral(stmt.LimitExpr.Pos().Line, stmt.LimitExpr.Pos().Column)
}
stmt.LimitExpr = expr
}
expr, err = p.analyzeExpression(ctx, stmt.HavingExpr, stmt)
if err != nil {
return nil, err

24
sql3/planner/executionplanner.go
View file

@ -69,6 +69,10 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen
rootOperator, err = p.compileSelectStatement(stmt, false)
case *parser.ShowDatabasesStatement:
rootOperator, err = p.compileShowDatabasesStatement(ctx, stmt)
case *parser.CopyStatement:
rootOperator, err = p.compileCopyStatement(stmt)
case *parser.PredictStatement:
rootOperator, err = p.compilePredictStatement(ctx, stmt)
case *parser.ShowTablesStatement:
rootOperator, err = p.compileShowTablesStatement(ctx, stmt)
case *parser.ShowColumnsStatement:
@ -93,16 +97,23 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen
rootOperator, err = p.compileDropTableStatement(ctx, stmt)
case *parser.DropViewStatement:
rootOperator, err = p.compileDropViewStatement(ctx, stmt)
case *parser.DropModelStatement:
rootOperator, err = p.compileDropModelStatement(stmt)
case *parser.InsertStatement:
rootOperator, err = p.compileInsertStatement(ctx, stmt)
case *parser.BulkInsertStatement:
rootOperator, err = p.compileBulkInsertStatement(ctx, stmt)
case *parser.DeleteStatement:
rootOperator, err = p.compileDeleteStatement(stmt)
case *parser.CreateModelStatement:
rootOperator, err = p.compileCreateModelStatement(stmt)
case *parser.CreateFunctionStatement:
rootOperator, err = p.compileCreateFunctionStatement(stmt)
default:
return nil, sql3.NewErrInternalf("cannot plan statement: %T", stmt)
}
// Optimize the plan.
// optimize the plan
if err == nil {
rootOperator, err = p.optimizePlan(ctx, rootOperator)
}
@ -130,6 +141,10 @@ func (p *ExecutionPlanner) analyzePlan(ctx context.Context, stmt parser.Statemen
return err
case *parser.ShowDatabasesStatement:
return nil
case *parser.CopyStatement:
return p.analyzeCopyStatement(ctx, stmt)
case *parser.PredictStatement:
return p.analyzePredictStatement(ctx, stmt)
case *parser.ShowTablesStatement:
return nil
case *parser.ShowColumnsStatement:
@ -154,12 +169,19 @@ func (p *ExecutionPlanner) analyzePlan(ctx context.Context, stmt parser.Statemen
return nil
case *parser.DropViewStatement:
return nil
case *parser.DropModelStatement:
return nil
case *parser.InsertStatement:
return p.analyzeInsertStatement(ctx, stmt)
case *parser.BulkInsertStatement:
return p.analyzeBulkInsertStatement(ctx, stmt)
case *parser.DeleteStatement:
return p.analyzeDeleteStatement(ctx, stmt)
case *parser.CreateModelStatement:
return p.analyzeCreateModelStatement(ctx, stmt)
case *parser.CreateFunctionStatement:
return p.analyzeCreateFunctionStatement(stmt)
default:
return sql3.NewErrInternalf("cannot analyze statement: %T", stmt)
}

58
sql3/planner/expression.go
View file

@ -1514,16 +1514,18 @@ func (n *inOpPlanExpression) WithChildren(children ...types.PlanExpression) (typ
// callPlanExpression is a function call
type callPlanExpression struct {
name string
args []types.PlanExpression
dataType parser.ExprDataType
name string
args []types.PlanExpression
dataType parser.ExprDataType
udfReference *functionSystemObject
}
func newCallPlanExpression(name string, args []types.PlanExpression, dataType parser.ExprDataType) *callPlanExpression {
func newCallPlanExpression(name string, args []types.PlanExpression, dataType parser.ExprDataType, udfReference *functionSystemObject) *callPlanExpression {
return &callPlanExpression{
name: name,
args: args,
dataType: dataType,
name: name,
args: args,
dataType: dataType,
udfReference: udfReference,
}
}
@ -1591,6 +1593,9 @@ func (n *callPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er
case "DATETIMEDIFF":
return n.EvaluateDatetimeDiff(currentRow)
default:
if n.udfReference != nil {
return n.evaluateUserDefinedFunction(currentRow)
}
return nil, sql3.NewErrInternalf("unhandled function name '%s'", n.name)
}
}
@ -1632,7 +1637,7 @@ func (n *callPlanExpression) WithChildren(children ...types.PlanExpression) (typ
if len(children) != len(n.args) {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newCallPlanExpression(n.name, children, n.dataType), nil
return newCallPlanExpression(n.name, children, n.dataType, n.udfReference), nil
}
// aliasPlanExpression is a alias ref
@ -2049,30 +2054,30 @@ func (n *sysVariablePlanExpression) WithChildren(children ...types.PlanExpressio
return n, nil
}
// dateLiteralPlanExpression is a date literal
type dateLiteralPlanExpression struct {
// timestampLiteralPlanExpression is a date literal
type timestampLiteralPlanExpression struct {
value time.Time
}
func newDateLiteralPlanExpression(value time.Time) *dateLiteralPlanExpression {
return &dateLiteralPlanExpression{
func newTimestampLiteralPlanExpression(value time.Time) *timestampLiteralPlanExpression {
return &timestampLiteralPlanExpression{
value: value,
}
}
func (n *dateLiteralPlanExpression) Evaluate(currentRow []interface{}) (interface{}, error) {
func (n *timestampLiteralPlanExpression) Evaluate(currentRow []interface{}) (interface{}, error) {
return n.value, nil
}
func (n *dateLiteralPlanExpression) Type() parser.ExprDataType {
func (n *timestampLiteralPlanExpression) Type() parser.ExprDataType {
return parser.NewDataTypeTimestamp()
}
func (n *dateLiteralPlanExpression) String() string {
func (n *timestampLiteralPlanExpression) String() string {
return n.value.Format(time.RFC3339Nano)
}
func (n *dateLiteralPlanExpression) Plan() map[string]interface{} {
func (n *timestampLiteralPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
result["description"] = n.String()
@ -2081,11 +2086,11 @@ func (n *dateLiteralPlanExpression) Plan() map[string]interface{} {
return result
}
func (n *dateLiteralPlanExpression) Children() []types.PlanExpression {
func (n *timestampLiteralPlanExpression) Children() []types.PlanExpression {
return []types.PlanExpression{}
}
func (n *dateLiteralPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
func (n *timestampLiteralPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
return n, nil
}
@ -2660,7 +2665,7 @@ func (p *ExecutionPlanner) compileExpr(expr parser.Expr) (_ types.PlanExpression
return newFloatLiteralPlanExpression(expr.Value), nil
case *parser.DateLit:
return newDateLiteralPlanExpression(expr.Value), nil
return newTimestampLiteralPlanExpression(expr.Value), nil
case *parser.SysVariable:
return newSysVariablePlanExpression(expr.Name(), expr.Token), nil
@ -2892,6 +2897,14 @@ func (p *ExecutionPlanner) compileCallExpr(expr *parser.Call) (_ types.PlanExpre
agg := newPercentilePlanExpression(args[0], args[1], expr.ResultDataType)
return agg, nil
case "CORR":
agg := newCorrPlanExpression(args[0], args[1], expr.ResultDataType)
return agg, nil
case "VAR":
agg := newVarPlanExpression(args[0], expr.ResultDataType)
return agg, nil
case "MIN":
agg := newMinPlanExpression(args[0], expr.ResultDataType)
return agg, nil
@ -2901,7 +2914,12 @@ func (p *ExecutionPlanner) compileCallExpr(expr *parser.Call) (_ types.PlanExpre
return agg, nil
default:
return newCallPlanExpression(parser.IdentName(expr.Name), args, expr.ResultDataType), nil
// could be a udf - try to look it up in functions
fn, err := p.getFunctionByName(strings.ToLower(callName))
if err != nil {
return nil, err
}
return newCallPlanExpression(parser.IdentName(expr.Name), args, expr.ResultDataType, fn), nil
}
}

4
sql3/planner/expression_it_test.go
View file

@ -40,7 +40,7 @@ func TestExpressions(t *testing.T) {
iop := newInOpPlanExpression(newIntLiteralPlanExpression(10), parser.IN, newIntLiteralPlanExpression(20))
assert.Equal(t, iop.String(), "10 in (20)")
callop := newCallPlanExpression("foo", []types.PlanExpression{newIntLiteralPlanExpression(10)}, parser.NewDataTypeInt())
callop := newCallPlanExpression("foo", []types.PlanExpression{newIntLiteralPlanExpression(10)}, parser.NewDataTypeInt(), nil)
assert.Equal(t, callop.String(), "foo(10)")
alop := newAliasPlanExpression("frobny", newIntLiteralPlanExpression(10))
@ -65,7 +65,7 @@ func TestExpressions(t *testing.T) {
assert.Equal(t, blop.String(), "false")
tm, _ := time.ParseInLocation(time.RFC3339, "2012-11-01T22:08:41+00:00", time.UTC)
dlop := newDateLiteralPlanExpression(tm)
dlop := newTimestampLiteralPlanExpression(tm)
assert.Equal(t, dlop.String(), "2012-11-01T22:08:41Z")
slop := newStringLiteralPlanExpression("foo")

306
sql3/planner/expressionagg.go
View file

@ -5,6 +5,7 @@ package planner
import (
"context"
"fmt"
"math"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/sql3"
@ -943,6 +944,311 @@ func (n *percentilePlanExpression) WithChildren(children ...types.PlanExpression
return newPercentilePlanExpression(children[0], children[1], n.returnDataType), nil
}
// aggregator for CORR()
type aggregateCorr struct {
expr *corrPlanExpression
n int64
sum_X float64
sum_Y float64
sum_XY float64
squareSum_X float64
squareSum_Y float64
}
func NewAggCorrBuffer(child *corrPlanExpression) *aggregateCorr {
return &aggregateCorr{
expr: child,
}
}
func (m *aggregateCorr) Update(ctx context.Context, row types.Row) error {
v1, err := m.expr.arg1.Evaluate(row)
if err != nil {
return err
}
v2, err := m.expr.arg2.Evaluate(row)
if err != nil {
return err
}
// skip if nil
if v1 == nil || v2 == nil {
return nil
}
var xVal float64
var yVal float64
switch dataType := m.expr.arg1.Type().(type) {
case *parser.DataTypeDecimal:
thisVal, ok := v1.(pql.Decimal)
if !ok {
return sql3.NewErrInternalf("unexpected type conversion '%T'", v1)
}
xVal = thisVal.Float64()
case *parser.DataTypeInt:
thisVal, ok := v1.(int64)
if !ok {
return sql3.NewErrInternalf("unexpected type conversion '%T'", v1)
}
xVal = float64(thisVal)
default:
return sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", dataType)
}
switch dataType := m.expr.arg2.Type().(type) {
case *parser.DataTypeDecimal:
thisVal, ok := v2.(pql.Decimal)
if !ok {
return sql3.NewErrInternalf("unexpected type conversion '%T'", v2)
}
yVal = thisVal.Float64()
case *parser.DataTypeInt:
thisVal, ok := v2.(int64)
if !ok {
return sql3.NewErrInternalf("unexpected type conversion '%T'", v2)
}
yVal = float64(thisVal)
default:
return sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", dataType)
}
m.sum_X = m.sum_X + xVal
m.sum_Y = m.sum_Y + yVal
m.sum_XY = m.sum_XY + xVal*yVal
m.squareSum_X = m.squareSum_X + xVal*xVal
m.squareSum_Y = m.squareSum_Y + yVal*yVal
m.n += 1
return nil
}
func (m *aggregateCorr) Eval(ctx context.Context) (interface{}, error) {
corr := float64((float64(m.n)*m.sum_XY - m.sum_X*m.sum_Y)) / (math.Sqrt(float64((float64(m.n)*m.squareSum_X - m.sum_X*m.sum_X) * (float64(m.n)*m.squareSum_Y - m.sum_Y*m.sum_Y))))
d, err := pql.FromFloat64WithScale(corr, 6)
if err != nil {
return nil, err
}
return d, nil
}
// corrPlanExpression handles CORR() - implement correlation coefficient
type corrPlanExpression struct {
arg1 types.PlanExpression
arg2 types.PlanExpression
returnDataType parser.ExprDataType
}
var _ types.Aggregable = (*corrPlanExpression)(nil)
func newCorrPlanExpression(arg1 types.PlanExpression, arg2 types.PlanExpression, returnDataType parser.ExprDataType) *corrPlanExpression {
return &corrPlanExpression{
arg1: arg1,
arg2: arg2,
returnDataType: returnDataType,
}
}
func (n *corrPlanExpression) Evaluate(currentRow []interface{}) (interface{}, error) {
return nil, sql3.NewErrInternalf("this should never be called")
}
func (n *corrPlanExpression) NewBuffer() (types.AggregationBuffer, error) {
return NewAggCorrBuffer(n), nil
}
func (n *corrPlanExpression) FirstChildExpr() types.PlanExpression {
return n.arg1
}
func (n *corrPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *corrPlanExpression) String() string {
return fmt.Sprintf("corr(%s, %s)", n.arg1.String(), n.arg2.String())
}
func (n *corrPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
result["description"] = n.String()
result["dataType"] = n.Type().TypeDescription()
result["arg1"] = n.arg1.Plan()
result["arg2"] = n.arg2.Plan()
return result
}
func (n *corrPlanExpression) Children() []types.PlanExpression {
return []types.PlanExpression{
n.arg1,
n.arg2,
}
}
func (n *corrPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
if len(children) != 2 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newCorrPlanExpression(children[0], children[1], n.returnDataType), nil
}
// aggregator for VAR()
type aggregateVar struct {
expr *varPlanExpression
// to calculate mean
n int64
sum float64
// we need to hang on to the values
// TODO(pok) - will need to spill these to disk for big result sets
values []float64
}
func NewAggVarBuffer(child *varPlanExpression) *aggregateVar {
return &aggregateVar{
expr: child,
values: make([]float64, 0),
}
}
func (m *aggregateVar) Update(ctx context.Context, row types.Row) error {
v, err := m.expr.arg.Evaluate(row)
if err != nil {
return err
}
// skip if nil
if v == nil {
return nil
}
var val float64
switch dataType := m.expr.arg.Type().(type) {
case *parser.DataTypeDecimal:
thisVal, ok := v.(pql.Decimal)
if !ok {
return sql3.NewErrInternalf("unexpected type conversion '%T'", v)
}
val = thisVal.Float64()
case *parser.DataTypeID:
thisVal, ok := v.(int64)
if !ok {
return sql3.NewErrInternalf("unexpected type conversion '%T'", v)
}
val = float64(thisVal)
case *parser.DataTypeInt:
thisVal, ok := v.(int64)
if !ok {
return sql3.NewErrInternalf("unexpected type conversion '%T'", v)
}
val = float64(thisVal)
default:
return sql3.NewErrInternalf("unhandled aggregate expression datatype '%T'", dataType)
}
m.sum += val
m.n += 1
m.values = append(m.values, val)
return nil
}
func (m *aggregateVar) Eval(ctx context.Context) (interface{}, error) {
mean := m.sum / float64(m.n)
var variance float64
for _, v := range m.values {
variance += (v - mean) * (v - mean)
}
variance = variance / float64(m.n)
d, err := pql.FromFloat64WithScale(variance, 6)
if err != nil {
return nil, err
}
return d, nil
}
// varPlanExpression handles VAR() - variance
type varPlanExpression struct {
arg types.PlanExpression
returnDataType parser.ExprDataType
}
var _ types.Aggregable = (*varPlanExpression)(nil)
func newVarPlanExpression(arg types.PlanExpression, returnDataType parser.ExprDataType) *varPlanExpression {
return &varPlanExpression{
arg: arg,
returnDataType: returnDataType,
}
}
func (n *varPlanExpression) Evaluate(currentRow []interface{}) (interface{}, error) {
return nil, sql3.NewErrInternalf("this should never be called")
}
func (n *varPlanExpression) NewBuffer() (types.AggregationBuffer, error) {
return NewAggVarBuffer(n), nil
}
func (n *varPlanExpression) FirstChildExpr() types.PlanExpression {
return n.arg
}
func (n *varPlanExpression) Type() parser.ExprDataType {
return n.returnDataType
}
func (n *varPlanExpression) String() string {
return fmt.Sprintf("var(%s)", n.arg.String())
}
func (n *varPlanExpression) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_expr"] = fmt.Sprintf("%T", n)
result["description"] = n.String()
result["dataType"] = n.Type().TypeDescription()
result["arg"] = n.arg.Plan()
return result
}
func (n *varPlanExpression) Children() []types.PlanExpression {
return []types.PlanExpression{
n.arg,
}
}
func (n *varPlanExpression) WithChildren(children ...types.PlanExpression) (types.PlanExpression, error) {
if len(children) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return newVarPlanExpression(children[0], n.returnDataType), nil
}
// aggregator for LAST()
type aggregateLast struct {
val interface{}

74
sql3/planner/expressionanalyzercall.go
View file

@ -124,6 +124,71 @@ func (p *ExecutionPlanner) analyzeCallExpression(ctx context.Context, call *pars
//return the data type of the referenced column
call.ResultDataType = ref.DataType()
case "CORR":
// can't do this on a *
if call.Star.IsValid() && len(call.Args) == 0 {
return nil, sql3.NewErrExpectedColumnReference(call.Star.Line, call.Star.Column)
}
if len(call.Args) != 2 {
return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 2, len(call.Args))
}
// if it is a ref, we shouldn't do a corr on the _id
arg1 := call.Args[0]
ref, ok := arg1.(*parser.QualifiedRef)
if ok && strings.EqualFold(ref.Column.Name, string(dax.PrimaryKeyFieldName)) {
return nil, sql3.NewErrIdColumnNotValidForAggregateFunction(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Name.Name)
}
// make sure the ref is the right type
if !(typeIsInteger(arg1.DataType()) || typeIsDecimal(arg1.DataType()) || typeIsTimestamp(arg1.DataType())) {
return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(arg1.Pos().Line, arg1.Pos().Column)
}
// if it is a ref, we shouldn't do a corr on the _id
arg2 := call.Args[1]
ref, ok = arg2.(*parser.QualifiedRef)
if ok && strings.EqualFold(ref.Column.Name, string(dax.PrimaryKeyFieldName)) {
return nil, sql3.NewErrIdColumnNotValidForAggregateFunction(call.Args[1].Pos().Line, call.Args[1].Pos().Column, call.Name.Name)
}
// make sure the ref is the right type
if !(typeIsInteger(arg2.DataType()) || typeIsDecimal(arg2.DataType()) || typeIsTimestamp(arg2.DataType())) {
return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(arg2.Pos().Line, arg2.Pos().Column)
}
// return the data type of the referenced column
call.ResultDataType = parser.NewDataTypeDecimal(6)
case "VAR":
// can't do this on a *
if call.Star.IsValid() && len(call.Args) == 0 {
return nil, sql3.NewErrExpectedColumnReference(call.Star.Line, call.Star.Column)
}
if len(call.Args) != 1 {
return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 1, len(call.Args))
}
// first arg should be a qualified ref
arg1 := call.Args[0]
ref, ok := arg1.(*parser.QualifiedRef)
if ok && strings.EqualFold(ref.Column.Name, string(dax.PrimaryKeyFieldName)) {
return nil, sql3.NewErrIdColumnNotValidForAggregateFunction(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Name.Name)
}
// make sure the ref is the right type
if !(typeIsInteger(arg1.DataType()) || typeIsDecimal(arg1.DataType()) || typeIsTimestamp(arg1.DataType())) {
return nil, sql3.NewErrIntOrDecimalOrTimestampExpressionExpected(arg1.Pos().Line, arg1.Pos().Column)
}
// return the data type of the referenced column
call.ResultDataType = parser.NewDataTypeDecimal(6)
case "MIN", "MAX":
// can't do an min/max on a *
if call.Star.IsValid() && len(call.Args) == 0 {
@ -270,6 +335,15 @@ func (p *ExecutionPlanner) analyzeCallExpression(ctx context.Context, call *pars
case "DATETIMEDIFF":
return p.analyzeFunctionDateTimeDiff(call, scope)
default:
// could be a udf - try to look it up in functions
fn, err := p.getFunctionByName(call.Name.Name)
if err != nil {
return nil, err
}
if fn != nil {
return p.analyzeUserDefinedFunction(call, scope, fn)
}
return nil, sql3.NewErrCallUnknownFunction(call.Name.NamePos.Line, call.Name.NamePos.Column, call.Name.Name)
}
return call, nil

2
sql3/planner/expressionpql.go
View file

@ -604,7 +604,7 @@ func planExprToValue(expr types.PlanExpression) (interface{}, error) {
return expr.value, nil
case *stringLiteralPlanExpression:
return expr.value, nil
case *dateLiteralPlanExpression:
case *timestampLiteralPlanExpression:
return expr.value, nil
case *boolLiteralPlanExpression:
return expr.value, nil

2
sql3/planner/opbulkinsert.go
View file

@ -867,7 +867,7 @@ func processColumnValue(rawValue interface{}, targetType parser.ExprDataType) (t
if !ok {
return nil, sql3.NewErrInternalf("unable to convert '%s", rawValue)
}
return newDateLiteralPlanExpression(tval), nil
return newTimestampLiteralPlanExpression(tval), nil
case *parser.DataTypeString:
sval, ok := rawValue.(string)

515
sql3/planner/opcopy.go Normal file
View file

@ -0,0 +1,515 @@
// Copyright 2022 Molecula Corp. All rights reserved.
package planner
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
// PlanOpCopy is a copy operator
type PlanOpCopy struct {
planner *ExecutionPlanner
targetTable string
url string
apiKey string
ddl string
ChildOp types.PlanOperator
warnings []string
}
func NewPlanOpCopy(planner *ExecutionPlanner, targetName string, url string, apiKey string, ddl string, child types.PlanOperator) *PlanOpCopy {
return &PlanOpCopy{
planner: planner,
targetTable: targetName,
url: url,
apiKey: apiKey,
ddl: ddl,
ChildOp: child,
warnings: make([]string, 0),
}
}
func (p *PlanOpCopy) Schema() types.Schema {
return types.Schema{}
}
func (p *PlanOpCopy) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) {
child, err := p.ChildOp.Iterator(ctx, row)
if err != nil {
return nil, err
}
if p.url != "" {
return newRemoteCopyIterator(p.planner, p.targetTable, p.url, p.apiKey, p.ddl, p.ChildOp.Schema(), child), nil
}
return newCopyIterator(p.planner, p.targetTable, p.ddl, p.ChildOp.Schema(), child), nil
}
func (p *PlanOpCopy) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) {
if len(children) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return NewPlanOpCopy(p.planner, p.targetTable, p.url, p.apiKey, p.ddl, children[0]), nil
}
func (p *PlanOpCopy) Children() []types.PlanOperator {
return []types.PlanOperator{
p.ChildOp,
}
}
func (p *PlanOpCopy) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_op"] = fmt.Sprintf("%T", p)
result["_schema"] = p.Schema().Plan()
result["child"] = p.ChildOp.Plan()
result["child"] = p.ChildOp.Plan()
return result
}
func (p *PlanOpCopy) String() string {
return ""
}
func (p *PlanOpCopy) AddWarning(warning string) {
p.warnings = append(p.warnings, warning)
}
func (p *PlanOpCopy) Warnings() []string {
return p.warnings
}
func (p *PlanOpCopy) Expressions() []types.PlanExpression {
return []types.PlanExpression{}
}
func (p *PlanOpCopy) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) {
if len(exprs) > 0 {
return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs))
}
return p, nil
}
type copyIterator struct {
planner *ExecutionPlanner
targetTableName string
copySchema types.Schema
ddl string
child types.RowIterator
hasStarted *struct{}
}
func newCopyIterator(planner *ExecutionPlanner, targetTableName string, ddl string, copySchema types.Schema, childIter types.RowIterator) *copyIterator {
return &copyIterator{
planner: planner,
targetTableName: targetTableName,
ddl: ddl,
copySchema: copySchema,
child: childIter,
}
}
func (i *copyIterator) Next(ctx context.Context) (types.Row, error) {
if i.hasStarted == nil {
// parse and execute the ddl to create the table
ast, err := parser.NewParser(strings.NewReader(i.ddl)).ParseStatement()
if err != nil {
return nil, err
}
ct, ok := ast.(*parser.CreateTableStatement)
if !ok {
return nil, sql3.NewErrInternalf("unexpected ast type")
}
// analyze
err = i.planner.analyzeCreateTableStatement(ct)
if err != nil {
return nil, err
}
ctOp, err := i.planner.compileCreateTableStatement(ctx, ct)
if err != nil {
return nil, err
}
ctIter, err := ctOp.Iterator(context.Background(), nil)
if err != nil {
return nil, err
}
_, err = ctIter.Next(ctx)
if err != nil && err != types.ErrNoMoreRows {
return nil, err
}
targetColumns := make([]*qualifiedRefPlanExpression, 0)
for _, s := range i.copySchema {
targetColumns = append(targetColumns, newQualifiedRefPlanExpression(i.targetTableName, s.ColumnName, 0, s.Type))
}
// build an insert iterator for the target table
insertIter := &insertRowIter{
planner: i.planner,
tableName: i.targetTableName,
targetColumns: targetColumns,
}
batchCount := 0
insertBatch := make([][]types.PlanExpression, 0)
for {
// get a source row
row, err := i.child.Next(ctx)
if err != nil {
if err == types.ErrNoMoreRows {
break
}
return nil, err
}
// add it to target batch
irow := make([]types.PlanExpression, len(row))
for i, s := range i.copySchema {
switch ty := s.Type.(type) {
case *parser.DataTypeID, *parser.DataTypeInt:
val, ok := row[i].(int64)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
irow[i] = newIntLiteralPlanExpression(val)
case *parser.DataTypeDecimal:
val, ok := row[i].(pql.Decimal)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
irow[i] = newFloatLiteralPlanExpression(val.String())
case *parser.DataTypeString:
val, ok := row[i].(string)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
irow[i] = newStringLiteralPlanExpression(val)
case *parser.DataTypeBool:
val, ok := row[i].(bool)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
irow[i] = newBoolLiteralPlanExpression(val)
case *parser.DataTypeTimestamp:
val, ok := row[i].(time.Time)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
irow[i] = newTimestampLiteralPlanExpression(val)
case *parser.DataTypeStringSet:
val, ok := row[i].([]string)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
members := make([]types.PlanExpression, 0)
for _, m := range val {
members = append(members, newStringLiteralPlanExpression(m))
}
irow[i] = newExprSetLiteralPlanExpression(members, parser.NewDataTypeStringSet())
case *parser.DataTypeIDSet:
val, ok := row[i].([]int64)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
members := make([]types.PlanExpression, 0)
for _, m := range val {
members = append(members, newIntLiteralPlanExpression(m))
}
irow[i] = newExprSetLiteralPlanExpression(members, parser.NewDataTypeIDSet())
default:
return nil, sql3.NewErrInternalf("unhandled type '%T'", ty)
}
}
insertBatch = append(insertBatch, irow)
// inc batch count
batchCount += 1
if batchCount > 1000 {
// do the insert
insertIter.insertValues = insertBatch
_, err = insertIter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return nil, err
}
// reset
batchCount = 0
insertBatch = make([][]types.PlanExpression, 0)
}
}
if len(insertBatch) > 0 {
// do the insert
insertIter.insertValues = insertBatch
_, err = insertIter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return nil, err
}
}
i.hasStarted = &struct{}{}
}
return nil, types.ErrNoMoreRows
}
type remoteCopyIterator struct {
planner *ExecutionPlanner
targetTableName string
copySchema types.Schema
url string
apiKey string
ddl string
child types.RowIterator
hasStarted *struct{}
}
func newRemoteCopyIterator(planner *ExecutionPlanner, targetTableName string, url string, apiKey string, ddl string, copySchema types.Schema, childIter types.RowIterator) *remoteCopyIterator {
return &remoteCopyIterator{
planner: planner,
targetTableName: targetTableName,
url: url,
apiKey: apiKey,
ddl: ddl,
copySchema: copySchema,
child: childIter,
}
}
func (i *remoteCopyIterator) remoteExec(ctx context.Context, sql string) (*pilosa.WireQueryResponse, error) {
// Create HTTP request.
req, err := http.NewRequest("POST", i.url, strings.NewReader(sql))
if err != nil {
return nil, sql3.NewErrInternalf("error executing remotely: %s", err.Error())
}
req.Header.Set("Content-Length", strconv.Itoa(len(sql)))
req.Header.Set("Content-Type", "text/plain")
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", "pilosa/"+i.planner.systemAPI.Version())
if len(i.apiKey) > 0 {
req.Header.Set("X-API-Key", i.apiKey)
}
// Execute request against the host.
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
if err != nil {
return nil, sql3.NewErrInternalf("error executing remotely: %s", err.Error())
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, sql3.NewErrInternalf("error executing remotely: %s", err.Error())
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
if resp.StatusCode == 401 {
return nil, sql3.NewErrRemoteUnauthorized(0, 0, i.url)
}
// we have an error
return nil, sql3.NewErrInternalf("error executing remotely: %d, %s", resp.StatusCode, string(body))
}
sqlResponse := &pilosa.WireQueryResponse{}
err = sqlResponse.UnmarshalJSONTyped([]byte(body), true)
if err != nil {
return nil, sql3.NewErrInternalf("error executing remotely: %s", err.Error())
}
if len(sqlResponse.Error) > 0 {
return nil, sql3.NewErrInternalf("error executing remotely: %s", sqlResponse.Error)
}
return sqlResponse, nil
}
func (i *remoteCopyIterator) Next(ctx context.Context) (types.Row, error) {
if i.hasStarted == nil {
// execute the ddl to create the table
_, err := i.remoteExec(ctx, i.ddl)
if err != nil {
return nil, err
}
// build bulk insert statement
var buf bytes.Buffer
buf.WriteString("bulk insert into ")
fmt.Fprintf(&buf, "%s", i.targetTableName)
buf.WriteString(" (")
for i, s := range i.copySchema {
if i > 0 {
buf.WriteString(", ")
}
fmt.Fprintf(&buf, "%s", s.ColumnName)
}
buf.WriteString(") map (")
for i, s := range i.copySchema {
if i > 0 {
buf.WriteString(", ")
}
fmt.Fprintf(&buf, "'$._%d' %s", i, s.Type.TypeDescription())
}
buf.WriteString(") from x'")
header := buf.String()
batchCount := 0
var batchBuf bytes.Buffer
for {
// get a source row
row, err := i.child.Next(ctx)
if err != nil {
if err == types.ErrNoMoreRows {
break
}
return nil, err
}
// add it to target batch
var rowBuf bytes.Buffer
rowBuf.WriteString("{")
for i, s := range i.copySchema {
if i > 0 {
rowBuf.WriteString(",")
}
fmt.Fprintf(&rowBuf, `"_%d":`, i)
if row[i] == nil {
rowBuf.WriteString("null")
continue
}
switch ty := s.Type.(type) {
case *parser.DataTypeID, *parser.DataTypeInt:
val, ok := row[i].(int64)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
fmt.Fprintf(&rowBuf, "%d", val)
case *parser.DataTypeString:
val, ok := row[i].(string)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
// escape single quotes
val = strings.ReplaceAll(val, `'`, `''`)
// and double quotes
val = strings.ReplaceAll(val, `"`, `\"`)
// and line feeds
if strings.Contains(val, "\n") {
val = strings.ReplaceAll(val, "\n", "\\n")
}
fmt.Fprintf(&rowBuf, `"%s"`, val)
case *parser.DataTypeBool:
val, ok := row[i].(bool)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
fmt.Fprintf(&rowBuf, "%v", val)
case *parser.DataTypeTimestamp:
val, ok := row[i].(time.Time)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
fmt.Fprintf(&rowBuf, `"%s"`, val.Format(time.RFC3339Nano))
case *parser.DataTypeStringSet:
val, ok := row[i].([]string)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
rowBuf.WriteString("[")
for j, s := range val {
if j > 0 {
rowBuf.WriteString(",")
}
fmt.Fprintf(&rowBuf, `"%s"`, s)
}
rowBuf.WriteString("]")
case *parser.DataTypeIDSet:
val, ok := row[i].([]int64)
if !ok {
return nil, sql3.NewErrInternalf("unexpected type '%T'", row[i])
}
rowBuf.WriteString("[")
for j, s := range val {
if j > 0 {
rowBuf.WriteString(",")
}
fmt.Fprintf(&rowBuf, `%d`, s)
}
rowBuf.WriteString("]")
default:
return nil, sql3.NewErrInternalf("unhandled type '%T'", ty)
}
}
rowBuf.WriteString("}\n")
batchBuf.Write(rowBuf.Bytes())
// inc batch count
batchCount += 1
if batchCount > 10000 {
// do the insert
var reqBuf bytes.Buffer
reqBuf.WriteString(header)
reqBuf.Write(batchBuf.Bytes())
reqBuf.WriteString("' with batchsize 10000 input 'STREAM' format 'NDJSON'")
_, err := i.remoteExec(ctx, reqBuf.String())
if err != nil {
return nil, err
}
// reset
batchCount = 0
batchBuf.Reset()
}
}
if batchCount > 0 {
// do the insert
var reqBuf bytes.Buffer
reqBuf.WriteString(header)
reqBuf.Write(batchBuf.Bytes())
reqBuf.WriteString("' with batchsize 10000 input 'STREAM' format 'NDJSON'")
_, err := i.remoteExec(ctx, reqBuf.String())
if err != nil {
return nil, err
}
}
i.hasStarted = &struct{}{}
}
return nil, types.ErrNoMoreRows
}

104
sql3/planner/opcreatefunction.go Normal file
View file

@ -0,0 +1,104 @@
// Copyright 2023 Molecula Corp. All rights reserved.
package planner
import (
"context"
"fmt"
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
// PlanOpCreateFunction implements the CREATE FUNCTION operator
type PlanOpCreateFunction struct {
planner *ExecutionPlanner
function *functionSystemObject
ifNotExists bool
warnings []string
}
func NewPlanOpCreateFunction(planner *ExecutionPlanner, ifNotExists bool, function *functionSystemObject) *PlanOpCreateFunction {
return &PlanOpCreateFunction{
planner: planner,
function: function,
ifNotExists: ifNotExists,
warnings: make([]string, 0),
}
}
func (p *PlanOpCreateFunction) Schema() types.Schema {
return types.Schema{}
}
func (p *PlanOpCreateFunction) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) {
return newCreateFunctionIter(p.planner, p.ifNotExists, p.function), nil
}
func (p *PlanOpCreateFunction) Children() []types.PlanOperator {
return []types.PlanOperator{}
}
func (p *PlanOpCreateFunction) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) {
if len(children) != 0 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return NewPlanOpCreateFunction(p.planner, p.ifNotExists, p.function), nil
}
func (p *PlanOpCreateFunction) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_op"] = fmt.Sprintf("%T", p)
result["_schema"] = p.Schema().Plan()
result["model"] = p.function.name
return result
}
func (p *PlanOpCreateFunction) String() string {
return ""
}
func (p *PlanOpCreateFunction) AddWarning(warning string) {
p.warnings = append(p.warnings, warning)
}
func (p *PlanOpCreateFunction) Warnings() []string {
var w []string
w = append(w, p.warnings...)
return w
}
type createFunctionIter struct {
planner *ExecutionPlanner
function *functionSystemObject
ifNotExists bool
}
func newCreateFunctionIter(planner *ExecutionPlanner, ifNotExists bool, function *functionSystemObject) *createFunctionIter {
return &createFunctionIter{
planner: planner,
function: function,
ifNotExists: ifNotExists,
}
}
func (i *createFunctionIter) Next(ctx context.Context) (types.Row, error) {
// now check in the functions table to see if it is exists
v, err := i.planner.getFunctionByName(i.function.name)
if err != nil {
return nil, err
}
if v != nil {
if i.ifNotExists {
return nil, types.ErrNoMoreRows
}
return nil, sql3.NewErrViewExists(0, 0, i.function.name)
}
// now store the view into fb_functions
err = i.planner.insertFunction(i.function)
if err != nil {
return nil, err
}
return nil, types.ErrNoMoreRows
}

279
sql3/planner/opcreatemodel.go Normal file
View file

@ -0,0 +1,279 @@
// Copyright 2022 Molecula Corp. All rights reserved.
package planner
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
uuid "github.com/satori/go.uuid"
)
// PlanOpCreateModel implements the CREATE MODEL operator
type PlanOpCreateModel struct {
ChildOp types.PlanOperator
planner *ExecutionPlanner
model *modelSystemObject
warnings []string
}
func NewPlanOpCreateModel(planner *ExecutionPlanner, model *modelSystemObject, child types.PlanOperator) *PlanOpCreateModel {
return &PlanOpCreateModel{
ChildOp: child,
planner: planner,
model: model,
warnings: make([]string, 0),
}
}
func (p *PlanOpCreateModel) Schema() types.Schema {
return types.Schema{}
}
func (p *PlanOpCreateModel) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) {
// get the query iterator
iter, err := p.ChildOp.Iterator(ctx, row)
if err != nil {
return nil, err
}
switch strings.ToLower(p.model.modelType) {
case "linear_regresssion":
return newCreateModelIter(p.planner, p.model, newLinearRegressionModelIter(p.planner, p.model, p.ChildOp.Schema(), iter)), nil
default:
return nil, sql3.NewErrInternalf("unexpected model tyoe '%s'", p.model.modelType)
}
}
func (p *PlanOpCreateModel) Children() []types.PlanOperator {
return []types.PlanOperator{
p.ChildOp,
}
}
func (p *PlanOpCreateModel) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) {
if len(children) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return NewPlanOpCreateModel(p.planner, p.model, children[0]), nil
}
func (p *PlanOpCreateModel) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_op"] = fmt.Sprintf("%T", p)
sc := make([]string, 0)
for _, e := range p.Schema() {
sc = append(sc, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeDescription()))
}
result["_schema"] = sc
result["model"] = p.model.name // TODO(pok) - add a Plan() method here (or some such)
result["child"] = p.ChildOp.Plan()
return result
}
func (p *PlanOpCreateModel) String() string {
return ""
}
func (p *PlanOpCreateModel) AddWarning(warning string) {
p.warnings = append(p.warnings, warning)
}
func (p *PlanOpCreateModel) Warnings() []string {
var w []string
w = append(w, p.warnings...)
w = append(w, p.ChildOp.Warnings()...)
return w
}
type createModelIter struct {
child types.RowIterator
planner *ExecutionPlanner
model *modelSystemObject
hasStarted *struct{}
}
func newCreateModelIter(planner *ExecutionPlanner, model *modelSystemObject, child types.RowIterator) *createModelIter {
return &createModelIter{
planner: planner,
model: model,
child: child,
}
}
func (i *createModelIter) Next(ctx context.Context) (types.Row, error) {
if i.hasStarted == nil {
// store the model into fb_models and set the model status to 'training'
i.model.status = "TRAINING"
err := i.planner.insertModel(i.model)
if err != nil {
return nil, err
}
// do the actual training
_, err = i.child.Next(ctx)
if err != nil && err != types.ErrNoMoreRows {
return nil, err
}
// update the model to ready
i.model.status = "READY"
err = i.planner.updateModel(i.model)
if err != nil {
return nil, err
}
i.hasStarted = &struct{}{}
}
return nil, types.ErrNoMoreRows
}
type linearRegressionModelIter struct {
child types.RowIterator
planner *ExecutionPlanner
model *modelSystemObject
childSchema types.Schema
hasStarted *struct{}
}
func newLinearRegressionModelIter(planner *ExecutionPlanner, model *modelSystemObject, childSchema types.Schema, child types.RowIterator) *linearRegressionModelIter {
return &linearRegressionModelIter{
planner: planner,
model: model,
childSchema: childSchema,
child: child,
}
}
func (i *linearRegressionModelIter) Next(ctx context.Context) (types.Row, error) {
if i.hasStarted == nil {
// this is linear regression now, so we actually 'train' when we predict (later)
// for now we just store the values from the query in fb_model_data
// delete anything from fb_model_data for this model
err := i.planner.ensureModelDataSystemTableExists()
if err != nil {
return nil, err
}
diter := &filteredDeleteRowIter{
planner: i.planner,
tableName: "fb_model_data",
filter: newBinOpPlanExpression(
newQualifiedRefPlanExpression("fb_model_data", "model_id", 0, parser.NewDataTypeString()),
parser.EQ,
newStringLiteralPlanExpression(i.model.name),
parser.NewDataTypeBool(),
),
}
_, err = diter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return nil, err
}
iter := &insertRowIter{
planner: i.planner,
tableName: "fb_model_data",
targetColumns: []*qualifiedRefPlanExpression{
newQualifiedRefPlanExpression("fb_model_data", "_id", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_model_data", "model_id", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_model_data", "data", 0, parser.NewDataTypeString()),
},
insertValues: [][]types.PlanExpression{},
}
trainingRefs := make([]*qualifiedRefPlanExpression, 0)
// make sure label column exists and is type compatible with float
labelColumn := i.model.labels[0]
found := false
for i, s := range i.childSchema {
if strings.EqualFold(labelColumn, s.ColumnName) {
if !typesAreAssignmentCompatible(parser.NewDataTypeDecimal(4), s.Type) {
return nil, sql3.NewErrInternalf("types not assignment compatible")
}
trainingRefs = append(trainingRefs, newQualifiedRefPlanExpression("", labelColumn, i, s.Type))
found = true
break
}
}
if !found {
return nil, sql3.NewErrInternalf("label column found found")
}
// make sure input columns exists and are type compatible with float
for _, ic := range i.model.inputColumns {
found := false
for i, s := range i.childSchema {
if strings.EqualFold(ic, s.ColumnName) {
if !typesAreAssignmentCompatible(parser.NewDataTypeDecimal(4), s.Type) {
return nil, sql3.NewErrInternalf("types not assignment compatible")
}
trainingRefs = append(trainingRefs, newQualifiedRefPlanExpression("", ic, i, s.Type))
found = true
break
}
}
if !found {
return nil, sql3.NewErrInternalf("input column found found")
}
}
// go run the query and iterate
for {
row, err := i.child.Next(ctx)
if err != nil {
if err == types.ErrNoMoreRows {
break
}
return nil, err
}
fdata := make([]float64, 0)
for _, ref := range trainingRefs {
val, err := ref.Evaluate(row)
if err != nil {
return nil, err
}
cval, err := coerceValue(ref.dataType, parser.NewDataTypeDecimal(4), val, parser.Pos{Line: 0, Column: 0})
if err != nil {
return nil, err
}
dval := cval.(pql.Decimal)
fdata = append(fdata, dval.Float64())
}
data, err := json.Marshal(fdata)
if err != nil {
return nil, err
}
rowID, err := uuid.NewV4()
if err != nil {
return nil, err
}
tuple := []types.PlanExpression{
newStringLiteralPlanExpression(rowID.String()),
newStringLiteralPlanExpression(i.model.name),
newStringLiteralPlanExpression(string(data)),
}
iter.insertValues = append(iter.insertValues, tuple)
fmt.Printf("%v", row)
}
_, err = iter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return nil, err
}
i.hasStarted = &struct{}{}
}
return nil, types.ErrNoMoreRows
}

102
sql3/planner/opdropmodel.go Normal file
View file

@ -0,0 +1,102 @@
// Copyright 2023 Molecula Corp. All rights reserved.
package planner
import (
"context"
"fmt"
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
// PlanOpDropModel plan operator to drop a view.
type PlanOpDropModel struct {
planner *ExecutionPlanner
modelName string
ifExists bool
warnings []string
}
func NewPlanOpDropModel(p *ExecutionPlanner, ifExists bool, modelName string) *PlanOpDropModel {
return &PlanOpDropModel{
planner: p,
modelName: modelName,
ifExists: ifExists,
warnings: make([]string, 0),
}
}
func (p *PlanOpDropModel) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_op"] = fmt.Sprintf("%T", p)
result["modelName"] = p.modelName
result["isExists"] = p.ifExists
return result
}
func (p *PlanOpDropModel) String() string {
return ""
}
func (p *PlanOpDropModel) AddWarning(warning string) {
p.warnings = append(p.warnings, warning)
}
func (p *PlanOpDropModel) Warnings() []string {
return p.warnings
}
func (p *PlanOpDropModel) Schema() types.Schema {
return types.Schema{}
}
func (p *PlanOpDropModel) Children() []types.PlanOperator {
return []types.PlanOperator{}
}
func (p *PlanOpDropModel) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) {
return &dropModelRowIter{
planner: p.planner,
ifExists: p.ifExists,
modelName: p.modelName,
}, nil
}
func (p *PlanOpDropModel) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) {
return nil, nil
}
type dropModelRowIter struct {
planner *ExecutionPlanner
ifExists bool
modelName string
}
var _ types.RowIterator = (*dropModelRowIter)(nil)
func (i *dropModelRowIter) Next(ctx context.Context) (types.Row, error) {
err := i.planner.checkAccess(ctx, i.modelName, accessTypeDropObject)
if err != nil {
return nil, err
}
// check in the models table to see if it exists
v, err := i.planner.getModelByName(i.modelName)
if err != nil {
return nil, err
}
if v == nil {
if i.ifExists {
return nil, types.ErrNoMoreRows
}
return nil, sql3.NewErrModelNotFound(0, 0, i.modelName)
}
err = i.planner.deleteModel(i.modelName)
if err != nil {
return nil, err
}
return nil, types.ErrNoMoreRows
}

253
sql3/planner/oppredict.go Normal file
View file

@ -0,0 +1,253 @@
// Copyright 2022 Molecula Corp. All rights reserved.
package planner
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
"github.com/sajari/regression"
)
// PlanOpPredict is an operator for a PREDICT
type PlanOpPredict struct {
ChildOp types.PlanOperator
planner *ExecutionPlanner
model *modelSystemObject
warnings []string
}
func NewPlanOpPredict(planner *ExecutionPlanner, model *modelSystemObject, child types.PlanOperator) *PlanOpPredict {
return &PlanOpPredict{
ChildOp: child,
planner: planner,
model: model,
warnings: make([]string, 0),
}
}
func (p *PlanOpPredict) Schema() types.Schema {
result := make(types.Schema, 0)
switch strings.ToLower(p.model.modelType) {
case "linear_regresssion":
labelName := p.model.labels[0]
result = append(result, &types.PlannerColumn{
ColumnName: fmt.Sprintf("predicted_%s", labelName),
RelationName: "",
AliasName: "",
// we need to get this type from somewhere...probably needs to be stored in the model def
Type: &parser.DataTypeDecimal{
Scale: 4,
},
})
default:
// don't add anything
}
// add the columns from the select
result = append(result, p.ChildOp.Schema()...)
return result
}
func (p *PlanOpPredict) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) {
// get the query iterator
iter, err := p.ChildOp.Iterator(ctx, row)
if err != nil {
return nil, err
}
switch strings.ToLower(p.model.modelType) {
case "linear_regresssion":
return newLinearRegressionPredictIter(p.planner, p.model, p.ChildOp.Schema(), iter), nil
default:
return nil, sql3.NewErrInternalf("unexpected model tyoe '%s'", p.model.modelType)
}
}
func (p *PlanOpPredict) Children() []types.PlanOperator {
return []types.PlanOperator{
p.ChildOp,
}
}
func (p *PlanOpPredict) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) {
if len(children) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return NewPlanOpPredict(p.planner, p.model, children[0]), nil
}
func (p *PlanOpPredict) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_op"] = fmt.Sprintf("%T", p)
sc := make([]string, 0)
for _, e := range p.Schema() {
sc = append(sc, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeDescription()))
}
result["_schema"] = sc
result["model"] = p.model.name
result["child"] = p.ChildOp.Plan()
return result
}
func (p *PlanOpPredict) String() string {
return ""
}
func (p *PlanOpPredict) AddWarning(warning string) {
p.warnings = append(p.warnings, warning)
}
func (p *PlanOpPredict) Warnings() []string {
var w []string
w = append(w, p.warnings...)
if p.ChildOp != nil {
w = append(w, p.ChildOp.Warnings()...)
}
return w
}
type linearRegressionPredictIter struct {
child types.RowIterator
planner *ExecutionPlanner
model *modelSystemObject
regres *regression.Regression
childSchema types.Schema
inferenceRefs []*qualifiedRefPlanExpression
hasStarted *struct{}
}
func newLinearRegressionPredictIter(planner *ExecutionPlanner, model *modelSystemObject, childSchema types.Schema, child types.RowIterator) *linearRegressionPredictIter {
return &linearRegressionPredictIter{
planner: planner,
model: model,
child: child,
childSchema: childSchema,
regres: new(regression.Regression),
inferenceRefs: make([]*qualifiedRefPlanExpression, 0),
}
}
func (i *linearRegressionPredictIter) Next(ctx context.Context) (types.Row, error) {
if i.hasStarted == nil {
// label column
i.regres.SetObserved("Murders per annum per 1,000,000 inhabitants")
// input columns
i.regres.SetVar(0, "Inhabitants")
i.regres.SetVar(1, "Percent with incomes below $5000")
i.regres.SetVar(2, "Percent unemployed")
// go get the 'training set' from fb_model_data
iter := &tableScanRowIter{
planner: i.planner,
tableName: "fb_model_data",
columns: []string{
"_id",
"model_id",
"data",
},
predicate: newBinOpPlanExpression(
newQualifiedRefPlanExpression("fb_model_data", "model_id", 0, parser.NewDataTypeString()),
parser.EQ,
newStringLiteralPlanExpression(i.model.name),
parser.NewDataTypeBool(),
),
}
for {
row, err := iter.Next(context.Background())
if err != nil {
if err == types.ErrNoMoreRows {
break
}
return nil, err
}
fdata := make([]float64, 0)
err = json.Unmarshal([]byte(row[2].(string)), &fdata)
if err != nil {
return nil, err
}
label := fdata[0]
vars := fdata[1:]
i.regres.Train(regression.DataPoint(label, vars))
}
// run the regression
err := i.regres.Run()
if err != nil {
return nil, err
}
fmt.Printf("Regression formula:\n%v\n", i.regres.Formula)
fmt.Printf("Regression:\n%s\n", i.regres)
for _, ic := range i.model.inputColumns {
found := false
for j, s := range i.childSchema {
if strings.EqualFold(ic, s.ColumnName) {
if !typesAreAssignmentCompatible(parser.NewDataTypeDecimal(4), s.Type) {
return nil, sql3.NewErrInternalf("types not assignment compatible")
}
i.inferenceRefs = append(i.inferenceRefs, newQualifiedRefPlanExpression("", ic, j, s.Type))
found = true
break
}
}
if !found {
return nil, sql3.NewErrInternalf("input column found found")
}
}
i.hasStarted = &struct{}{}
}
childrow, err := i.child.Next(ctx)
if err != nil {
return nil, err
}
// construct the inference data
inferenceData := make([]float64, len(i.inferenceRefs))
for j, ref := range i.inferenceRefs {
val, err := ref.Evaluate(childrow)
if err != nil {
return nil, err
}
cval, err := coerceValue(ref.dataType, parser.NewDataTypeDecimal(4), val, parser.Pos{Line: 0, Column: 0})
if err != nil {
return nil, err
}
dval := cval.(pql.Decimal)
inferenceData[j] = dval.Float64()
}
// do the prediction
prediction, err := i.regres.Predict(inferenceData)
if err != nil {
return nil, err
}
// turn the predition into a decimal
dprediction, err := pql.FromFloat64WithScale(prediction, 4)
if err != nil {
return nil, err
}
// make an output row
row := make(types.Row, len(childrow)+1)
row[0] = dprediction
copy(row[1:], childrow)
return row, nil
}

195
sql3/planner/opsystemtable.go
View file

@ -9,6 +9,7 @@ import (
"sort"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
@ -500,6 +501,107 @@ type fbTableDDLRowIter struct {
var _ types.RowIterator = (*fbTableDDLRowIter)(nil)
func generateTableDDL(tbl *dax.Table, newName string) string {
var buf bytes.Buffer
buf.WriteString("create table ")
if len(newName) > 0 {
fmt.Fprintf(&buf, "%s", newName)
} else {
fmt.Fprintf(&buf, "%s", tbl.Name)
}
buf.WriteString(" (")
for idx, col := range tbl.Fields {
if idx > 0 {
buf.WriteString(", ")
}
fmt.Fprintf(&buf, "%s", col.Name)
dataType := fieldSQLDataType(pilosa.FieldToFieldInfo(col))
fmt.Fprintf(&buf, " %s", dataType.TypeDescription())
switch dt := dataType.(type) {
case *parser.DataTypeID, *parser.DataTypeString:
if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 {
// if we still have the default, we need to print that out if we have a non-default size
if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
fmt.Fprintf(&buf, " size %d", col.Options.CacheSize)
}
case *parser.DataTypeIDSet, *parser.DataTypeStringSet:
if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 {
// if we still have the default, we need to print that out if we have a non-default size
if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
fmt.Fprintf(&buf, " size %d", col.Options.CacheSize)
}
case *parser.DataTypeIDSetQuantum, *parser.DataTypeStringSetQuantum:
if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 {
// if we still have the default, we need to print that out if we have a non-default size
if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
fmt.Fprintf(&buf, " size %d", col.Options.CacheSize)
}
if !col.Options.TimeQuantum.IsEmpty() {
fmt.Fprintf(&buf, " timequantum '%s'", col.Options.TimeQuantum)
}
if col.Options.TTL > 0 {
fmt.Fprintf(&buf, " ttl '%s'", col.Options.TTL.String())
}
case *parser.DataTypeInt:
minValue, maxValue := pql.MinMax(0)
min := col.Options.Min
if !min.EqualTo(minValue) {
fmt.Fprintf(&buf, " min %d", min.ToInt64(0))
}
max := col.Options.Max
if !max.EqualTo(maxValue) {
fmt.Fprintf(&buf, " max %d", max.ToInt64(0))
}
case *parser.DataTypeDecimal:
minValue, maxValue := pql.MinMax(dt.Scale)
min := col.Options.Min
if !min.EqualTo(minValue) {
fmt.Fprintf(&buf, " min %v", min)
}
max := col.Options.Max
if !max.EqualTo(maxValue) {
fmt.Fprintf(&buf, " max %v", max)
}
case *parser.DataTypeTimestamp:
if len(col.Options.TimeUnit) > 0 {
fmt.Fprintf(&buf, " timeunit '%s'", col.Options.TimeUnit)
}
// TODO(pok) how do we get epoch out of col?
}
}
buf.WriteString(");")
return buf.String()
}
func (i *fbTableDDLRowIter) Next(ctx context.Context) (types.Row, error) {
if i.result == nil {
tbls, err := i.planner.schemaAPI.Tables(ctx)
@ -512,98 +614,7 @@ func (i *fbTableDDLRowIter) Next(ctx context.Context) (types.Row, error) {
for idx, tbl := range tbls {
// build the ddl for this table
var buf bytes.Buffer
buf.WriteString("create table ")
fmt.Fprintf(&buf, "%s", tbl.Name)
buf.WriteString(" (")
for idx, col := range tbl.Fields {
if idx > 0 {
buf.WriteString(", ")
}
fmt.Fprintf(&buf, "%s", col.Name)
dataType := fieldSQLDataType(pilosa.FieldToFieldInfo(col))
fmt.Fprintf(&buf, " %s", dataType.TypeDescription())
switch dt := dataType.(type) {
case *parser.DataTypeID, *parser.DataTypeString:
if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 {
// if we still have the default, we need to print that out if we have a non-default size
if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
fmt.Fprintf(&buf, " size %d", col.Options.CacheSize)
}
case *parser.DataTypeIDSet, *parser.DataTypeStringSet:
if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 {
// if we still have the default, we need to print that out if we have a non-default size
if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
fmt.Fprintf(&buf, " size %d", col.Options.CacheSize)
}
case *parser.DataTypeIDSetQuantum, *parser.DataTypeStringSetQuantum:
if col.Options.CacheType != pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
if col.Options.CacheSize != pilosa.DefaultCacheSize && col.Options.CacheSize > 0 {
// if we still have the default, we need to print that out if we have a non-default size
if col.Options.CacheType == pilosa.DefaultCacheType && len(col.Options.CacheType) > 0 {
fmt.Fprintf(&buf, " cachetype %s", col.Options.CacheType)
}
fmt.Fprintf(&buf, " size %d", col.Options.CacheSize)
}
if !col.Options.TimeQuantum.IsEmpty() {
fmt.Fprintf(&buf, " timequantum '%s'", col.Options.TimeQuantum)
}
if col.Options.TTL > 0 {
fmt.Fprintf(&buf, " ttl '%s'", col.Options.TTL.String())
}
case *parser.DataTypeInt:
minValue, maxValue := pql.MinMax(0)
min := col.Options.Min
if !min.EqualTo(minValue) {
fmt.Fprintf(&buf, " min %d", min.ToInt64(0))
}
max := col.Options.Max
if !max.EqualTo(maxValue) {
fmt.Fprintf(&buf, " max %d", max.ToInt64(0))
}
case *parser.DataTypeDecimal:
minValue, maxValue := pql.MinMax(dt.Scale)
min := col.Options.Min
if !min.EqualTo(minValue) {
fmt.Fprintf(&buf, " min %v", min)
}
max := col.Options.Max
if !max.EqualTo(maxValue) {
fmt.Fprintf(&buf, " max %v", max)
}
case *parser.DataTypeTimestamp:
if len(col.Options.TimeUnit) > 0 {
fmt.Fprintf(&buf, " timeunit '%s'", col.Options.TimeUnit)
}
// TODO(pok) how do we get epoch out of col?
}
}
buf.WriteString(");")
ddl := buf.String()
ddl := generateTableDDL(tbl, "")
i.result[idx] = &fbTableDDLRow{
id: string(tbl.Name),

8
sql3/planner/planoptimizer.go
View file

@ -1,4 +1,4 @@
// Copyright 2021 Molecula Corp. All rights reserved.
// Copyright 2023 Molecula Corp. All rights reserved.
package planner
@ -14,6 +14,8 @@ import (
"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
)
//TODO(pok) give every expression an id 'Expr1234' and use that to match on
//TODO(pok) have a rule to eliminate PlanOpRelAlias
//TODO(pok) push filter down into join condition if terms reference either side of join
//TODO(pok) push order by down as far as possible
//TODO(pok) you can't group by _id in PQL, so we need to not use a PQL group by operator here
@ -707,6 +709,10 @@ func tryToReplaceGroupByWithPQLAggregate(ctx context.Context, a *ExecutionPlanne
}
thisNode.Aggregates[i] = newAgg
// these two can't be done in PQL
case *corrPlanExpression, *varPlanExpression:
return thisNode, true, nil
case types.Aggregable:
switch ref := aggregable.FirstChildExpr().(type) {
case *qualifiedRefPlanExpression:

639
sql3/planner/systemobjects.go
View file

@ -4,6 +4,7 @@ package planner
import (
"context"
"encoding/json"
"time"
pilosa "github.com/featurebasedb/featurebase/v3"
@ -18,6 +19,20 @@ type viewSystemObject struct {
statement string
}
type functionSystemObject struct {
name string
language string
body string
}
type modelSystemObject struct {
name string
status string
modelType string
labels []string
inputColumns []string
}
func (p *ExecutionPlanner) ensureViewsSystemTableExists(ctx context.Context) error {
_, err := p.schemaAPI.TableByName(ctx, "fb_views")
if err != nil {
@ -178,8 +193,8 @@ func (p *ExecutionPlanner) insertView(ctx context.Context, view *viewSystemObjec
newStringLiteralPlanExpression(view.statement),
newStringLiteralPlanExpression(""),
newStringLiteralPlanExpression(""),
newDateLiteralPlanExpression(createTime),
newDateLiteralPlanExpression(createTime),
newTimestampLiteralPlanExpression(createTime),
newTimestampLiteralPlanExpression(createTime),
},
},
}
@ -212,7 +227,7 @@ func (p *ExecutionPlanner) updateView(ctx context.Context, view *viewSystemObjec
newStringLiteralPlanExpression(view.name),
newStringLiteralPlanExpression(view.statement),
newStringLiteralPlanExpression(""),
newDateLiteralPlanExpression(updateTime),
newTimestampLiteralPlanExpression(updateTime),
},
},
}
@ -245,3 +260,621 @@ func (p *ExecutionPlanner) deleteView(ctx context.Context, viewName string) erro
}
return nil
}
func (p *ExecutionPlanner) ensureFunctionsSystemTableExists() error {
_, err := p.schemaAPI.TableByName(context.Background(), "fb_functions")
if err != nil {
if !isTableNotFoundError(err) {
return err
}
// create table fb_functions (
// _id string
// name string
// language string
// body string
// owner string
// updated_by string
// created_at timestamp
// updated_at timestamp
// );
// if it doesn't, create it by making the appropriate iterator
iter := &createTableRowIter{
planner: p,
tableName: "fb_functions",
failIfExists: false,
isKeyed: true,
keyPartitions: 0,
columns: []*createTableField{
{
planner: p,
name: "name",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "language",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "body",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "owner",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "updated_by",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "created_at",
typeName: dax.BaseTypeTimestamp,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds),
},
},
{
planner: p,
name: "updated_at",
typeName: dax.BaseTypeTimestamp,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds),
},
},
},
description: "system table for functions",
}
// call next on our iterator to create the table
_, err := iter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return err
}
}
return nil
}
func (p *ExecutionPlanner) getFunctionByName(name string) (*functionSystemObject, error) {
err := p.ensureFunctionsSystemTableExists()
if err != nil {
return nil, err
}
tbl, err := p.schemaAPI.TableByName(context.Background(), "fb_functions")
if err != nil {
return nil, sql3.NewErrTableNotFound(0, 0, "fb_functions")
}
cols := make([]string, len(tbl.Fields))
for i, c := range tbl.Fields {
cols[i] = string(c.Name)
}
iter := &tableScanRowIter{
planner: p,
tableName: "fb_functions",
columns: cols,
predicate: newBinOpPlanExpression(
newQualifiedRefPlanExpression("fb_functions", "_id", 0, parser.NewDataTypeString()),
parser.EQ,
newStringLiteralPlanExpression(name),
parser.NewDataTypeBool(),
),
topExpr: nil,
}
row, err := iter.Next(context.Background())
if err != nil {
if err == types.ErrNoMoreRows {
// view does not exist
return nil, nil
}
return nil, err
}
return &functionSystemObject{
name: row[1].(string),
language: row[2].(string),
body: row[3].(string),
}, nil
}
func (p *ExecutionPlanner) insertFunction(function *functionSystemObject) error {
err := p.ensureFunctionsSystemTableExists()
if err != nil {
return err
}
createTime := time.Now().UTC()
iter := &insertRowIter{
planner: p,
tableName: "fb_functions",
targetColumns: []*qualifiedRefPlanExpression{
newQualifiedRefPlanExpression("fb_functions", "_id", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_functions", "name", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_functions", "language", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_functions", "body", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_functions", "owner", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_functions", "updated_by", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_functions", "created_at", 0, parser.NewDataTypeTimestamp()),
newQualifiedRefPlanExpression("fb_functions", "updated_at", 0, parser.NewDataTypeTimestamp()),
},
insertValues: [][]types.PlanExpression{
{
newStringLiteralPlanExpression(function.name),
newStringLiteralPlanExpression(function.name),
newStringLiteralPlanExpression(function.language),
newStringLiteralPlanExpression(function.body),
newStringLiteralPlanExpression(""),
newStringLiteralPlanExpression(""),
newTimestampLiteralPlanExpression(createTime),
newTimestampLiteralPlanExpression(createTime),
},
},
}
_, err = iter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return err
}
return nil
}
func (p *ExecutionPlanner) updateFunction(function *functionSystemObject) error {
err := p.ensureFunctionsSystemTableExists()
if err != nil {
return err
}
updateTime := time.Now().UTC()
iter := &insertRowIter{
planner: p,
tableName: "fb_functions",
targetColumns: []*qualifiedRefPlanExpression{
newQualifiedRefPlanExpression("fb_functions", "_id", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_functions", "language", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_functions", "body", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_functions", "updated_by", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_functions", "updated_at", 0, parser.NewDataTypeTimestamp()),
},
insertValues: [][]types.PlanExpression{
{
newStringLiteralPlanExpression(function.name),
newStringLiteralPlanExpression(function.language),
newStringLiteralPlanExpression(function.body),
newStringLiteralPlanExpression(""),
newTimestampLiteralPlanExpression(updateTime),
},
},
}
_, err = iter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return err
}
return nil
}
func (p *ExecutionPlanner) deleteFunction(functionName string) error {
err := p.ensureFunctionsSystemTableExists()
if err != nil {
return err
}
iter := &filteredDeleteRowIter{
planner: p,
tableName: "fb_functions",
filter: newBinOpPlanExpression(
newQualifiedRefPlanExpression("fb_functions", "_id", 0, parser.NewDataTypeString()),
parser.EQ,
newStringLiteralPlanExpression(functionName),
parser.NewDataTypeBool(),
),
}
_, err = iter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return err
}
return nil
}
func (p *ExecutionPlanner) ensureModelsSystemTableExists() error {
_, err := p.schemaAPI.TableByName(context.Background(), "fb_models")
if err != nil {
if !isTableNotFoundError(err) {
return err
}
// create table fb_models (
// _id string
// name string
// status string
// model_type string
// labels string --this is an array of string, we'll store it as a json object until we have string[] type in sql
// input_columns string --this is an array for string, we'll store it as a json object until we have string[] type in sql
// owner string
// updated_by string
// created_at timestamp
// updated_at timestamp
// );
// if it doesn't, create it by making the appropriate iterator
iter := &createTableRowIter{
planner: p,
tableName: "fb_models",
failIfExists: false,
isKeyed: true,
keyPartitions: 0,
columns: []*createTableField{
{
planner: p,
name: "name",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "status",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "model_type",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "labels",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "input_columns",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "owner",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "updated_by",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "created_at",
typeName: dax.BaseTypeTimestamp,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds),
},
},
{
planner: p,
name: "updated_at",
typeName: dax.BaseTypeTimestamp,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds),
},
},
},
description: "system table for models",
}
// call next on our iterator to create the table
_, err := iter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return err
}
}
return nil
}
func (p *ExecutionPlanner) ensureModelDataSystemTableExists() error {
_, err := p.schemaAPI.TableByName(context.Background(), "fb_model_data")
if err != nil {
if !isTableNotFoundError(err) {
return err
}
// create table fb_model_data (
// _id string
// model_id string
// data string
// );
// if it doesn't, create it by making the appropriate iterator
iter := &createTableRowIter{
planner: p,
tableName: "fb_model_data",
failIfExists: false,
isKeyed: true,
keyPartitions: 0,
columns: []*createTableField{
{
planner: p,
name: "model_id",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
{
planner: p,
name: "data",
typeName: dax.BaseTypeString,
fos: []pilosa.FieldOption{
pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize),
pilosa.OptFieldKeys(),
},
},
},
description: "system table for model data",
}
// call next on our iterator to create the table
_, err := iter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return err
}
}
return nil
}
func (p *ExecutionPlanner) getModelByName(name string) (*modelSystemObject, error) {
err := p.ensureModelsSystemTableExists()
if err != nil {
return nil, err
}
tbl, err := p.schemaAPI.TableByName(context.Background(), "fb_models")
if err != nil {
return nil, sql3.NewErrTableNotFound(0, 0, "fb_models")
}
cols := make([]string, len(tbl.Fields))
for i, c := range tbl.Fields {
cols[i] = string(c.Name)
}
iter := &tableScanRowIter{
planner: p,
tableName: "fb_models",
columns: cols,
predicate: newBinOpPlanExpression(
newQualifiedRefPlanExpression("fb_models", "_id", 0, parser.NewDataTypeString()),
parser.EQ,
newStringLiteralPlanExpression(name),
parser.NewDataTypeBool(),
),
topExpr: nil,
}
row, err := iter.Next(context.Background())
if err != nil {
if err == types.ErrNoMoreRows {
// model does not exist
return nil, nil
}
return nil, err
}
labels := make([]string, 0)
err = json.Unmarshal([]byte(row[4].(string)), &labels)
if err != nil {
return nil, err
}
inputColumns := make([]string, 0)
err = json.Unmarshal([]byte(row[5].(string)), &inputColumns)
if err != nil {
return nil, err
}
return &modelSystemObject{
name: row[1].(string),
status: row[2].(string),
modelType: row[3].(string),
labels: labels,
inputColumns: inputColumns,
}, nil
}
func (p *ExecutionPlanner) insertModel(model *modelSystemObject) error {
err := p.ensureModelsSystemTableExists()
if err != nil {
return err
}
createTime := time.Now().UTC()
labelJson, err := json.Marshal(model.labels)
if err != nil {
return err
}
inputColumnsJson, err := json.Marshal(model.inputColumns)
if err != nil {
return err
}
iter := &insertRowIter{
planner: p,
tableName: "fb_models",
targetColumns: []*qualifiedRefPlanExpression{
newQualifiedRefPlanExpression("fb_models", "_id", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "name", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "status", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "model_type", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "labels", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "input_columns", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "owner", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "updated_by", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "created_at", 0, parser.NewDataTypeTimestamp()),
newQualifiedRefPlanExpression("fb_models", "updated_at", 0, parser.NewDataTypeTimestamp()),
},
insertValues: [][]types.PlanExpression{
{
newStringLiteralPlanExpression(model.name),
newStringLiteralPlanExpression(model.name),
newStringLiteralPlanExpression(model.status),
newStringLiteralPlanExpression(model.modelType),
newStringLiteralPlanExpression(string(labelJson)),
newStringLiteralPlanExpression(string(inputColumnsJson)),
newStringLiteralPlanExpression(""),
newStringLiteralPlanExpression(""),
newTimestampLiteralPlanExpression(createTime),
newTimestampLiteralPlanExpression(createTime),
},
},
}
_, err = iter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return err
}
return nil
}
func (p *ExecutionPlanner) updateModel(model *modelSystemObject) error {
err := p.ensureModelsSystemTableExists()
if err != nil {
return err
}
updateTime := time.Now().UTC()
labelJson, err := json.Marshal(model.labels)
if err != nil {
return err
}
inputColumnsJson, err := json.Marshal(model.inputColumns)
if err != nil {
return err
}
iter := &insertRowIter{
planner: p,
tableName: "fb_models",
targetColumns: []*qualifiedRefPlanExpression{
newQualifiedRefPlanExpression("fb_models", "_id", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "status", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "model_type", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "labels", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "input_columns", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "updated_by", 0, parser.NewDataTypeString()),
newQualifiedRefPlanExpression("fb_models", "updated_at", 0, parser.NewDataTypeTimestamp()),
},
insertValues: [][]types.PlanExpression{
{
newStringLiteralPlanExpression(model.name),
newStringLiteralPlanExpression(model.status),
newStringLiteralPlanExpression(model.modelType),
newStringLiteralPlanExpression(string(labelJson)),
newStringLiteralPlanExpression(string(inputColumnsJson)),
newStringLiteralPlanExpression(""),
newTimestampLiteralPlanExpression(updateTime),
},
},
}
_, err = iter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return err
}
return nil
}
func (p *ExecutionPlanner) deleteModel(modelName string) error {
err := p.ensureModelsSystemTableExists()
if err != nil {
return err
}
err = p.ensureModelDataSystemTableExists()
if err != nil {
return err
}
iter := &filteredDeleteRowIter{
planner: p,
tableName: "fb_models",
filter: newBinOpPlanExpression(
newQualifiedRefPlanExpression("fb_models", "_id", 0, parser.NewDataTypeString()),
parser.EQ,
newStringLiteralPlanExpression(modelName),
parser.NewDataTypeBool(),
),
}
_, err = iter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return err
}
iter = &filteredDeleteRowIter{
planner: p,
tableName: "fb_model_data",
filter: newBinOpPlanExpression(
newQualifiedRefPlanExpression("fb_model_data", "model_id", 0, parser.NewDataTypeString()),
parser.EQ,
newStringLiteralPlanExpression(modelName),
parser.NewDataTypeBool(),
),
}
_, err = iter.Next(context.Background())
if err != nil && err != types.ErrNoMoreRows {
return err
}
return nil
}

71
sql3/planner/userdefinedfunctions.go Normal file
View file

@ -0,0 +1,71 @@
package planner
import (
"github.com/featurebasedb/featurebase/v3/sql3"
"github.com/featurebasedb/featurebase/v3/sql3/parser"
)
func (p *ExecutionPlanner) analyzeUserDefinedFunction(call *parser.Call, scope parser.Statement, function *functionSystemObject) (parser.Expr, error) {
// TODO(pok) removing user defined functions for now
// // hard code to 1 string parameter
// if len(call.Args) != 1 {
// return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 2, len(call.Args))
// }
// // arg 1
// argType := parser.NewDataTypeString()
// if !typesAreAssignmentCompatible(argType, call.Args[0].DataType()) {
// return nil, sql3.NewErrParameterTypeMistmatch(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Args[0].DataType().TypeDescription(), argType.TypeDescription())
// }
// //return string
// call.ResultDataType = parser.NewDataTypeString()
// return call, nil
return nil, sql3.NewErrUnsupported(0, 0, false, "user defined functions")
}
func (n *callPlanExpression) evaluateUserDefinedFunction(currentRow []interface{}) (interface{}, error) {
// TODO(pok) removing what effectively is a remote code exploit
// we will come back to this to add sql udfs and external code later
// argEval, err := n.args[0].Evaluate(currentRow)
// if err != nil {
// return nil, err
// }
// // nil if anything is nil
// if argEval == nil {
// return nil, nil
// }
// //get the value
// coercedArg, err := coerceValue(n.args[0].Type(), parser.NewDataTypeString(), argEval, parser.Pos{Line: 0, Column: 0})
// if err != nil {
// return nil, err
// }
// arg, argOk := coercedArg.(string)
// if !argOk {
// return nil, sql3.NewErrInternalf("unable to convert value")
// }
// // save the body to a temp file
// file, err := os.CreateTemp("", "py-body")
// if err != nil {
// return nil, err
// }
// defer os.Remove(file.Name())
// file.Write([]byte(n.udfReference.body))
// cmd := exec.Command("python3", file.Name(), arg)
// stdout, err := cmd.Output()
// if err != nil {
// return nil, err
// }
// retVal := string(stdout)
// return retVal, nil
return nil, sql3.NewErrUnsupported(0, 0, false, "user defined functions")
}

8
sql3/test/defs/defs.go
View file

@ -39,7 +39,7 @@ var TableTests []TableTest = []TableTest{
subqueryTests,
viewTests,
topTests,
topLimitTests,
deleteTests,
@ -179,6 +179,8 @@ var TableTests []TableTest = []TableTest{
avgTests,
percentileTests,
minmaxTests,
corrTests,
varTests,
// groupby tests
groupByTests,
@ -197,6 +199,10 @@ var TableTests []TableTest = []TableTest{
bulkInsertTable,
bulkInsert,
// copy
copyTable,
copyTests,
// bool (batch logic)
boolTests,

175
sql3/test/defs/defs_aggregate.go
View file

@ -737,3 +737,178 @@ var minmaxTests = TableTest{
},
},
}
var corrTests = TableTest{
Table: tbl(
"corr_test",
srcHdrs(
srcHdr("_id", fldTypeID),
srcHdr("i1", fldTypeInt, "min 0", "max 1000"),
srcHdr("d1", fldTypeDecimal2),
srcHdr("s1", fldTypeString),
srcHdr("id1", fldTypeID),
),
srcRows(
srcRow(int64(1), int64(10), float64(10), string("foo"), int64(10)),
srcRow(int64(2), int64(10), float64(10), string("foo2"), int64(11)),
srcRow(int64(3), int64(11), float64(11), string("foo34"), int64(12)),
srcRow(int64(4), int64(12), float64(12), string("foo45"), int64(13)),
srcRow(int64(5), int64(12), float64(12), string("foo63"), int64(14)),
srcRow(int64(6), int64(13), float64(13), string("foo22222"), int64(15)),
),
),
SQLTests: []SQLTest{
{
SQLs: sqls(
"SELECT corr(*, i1) AS corr1 FROM corr_test",
),
ExpErr: "expected right paren, found ','",
},
{
SQLs: sqls(
"SELECT corr(i1, d1) AS corr1 FROM corr_test",
),
ExpHdrs: hdrs(
hdr("corr1", featurebase.WireQueryField{
Type: dax.BaseTypeDecimal + "(6)",
BaseType: dax.BaseTypeDecimal,
TypeInfo: map[string]interface{}{"scale": int64(6)},
}),
),
ExpRows: rows(
row(pql.NewDecimal(1000000, 6)),
),
Compare: CompareExactUnordered,
},
{
SQLs: sqls(
"SELECT corr(_id, i1) AS corr1 FROM corr_test",
),
ExpErr: "_id column cannot be used in aggregate function 'corr'",
},
{
SQLs: sqls(
"SELECT corr(i1) AS corr1 FROM corr_test",
),
ExpErr: "count of formal parameters (2) does not match count of actual parameters (1)",
},
{
SQLs: sqls(
"SELECT corr(s1, i1) AS avg_rows FROM corr_test",
),
ExpErr: "integer, decimal or timestamp expression expected",
},
{
SQLs: sqls(
"SELECT corr(len(s1), i1) AS corr1 FROM corr_test",
),
ExpHdrs: hdrs(
hdr("corr1", featurebase.WireQueryField{
Type: dax.BaseTypeDecimal + "(6)",
BaseType: dax.BaseTypeDecimal,
TypeInfo: map[string]interface{}{"scale": int64(6)},
}),
),
ExpRows: rows(
row(pql.NewDecimal(888234, 6)),
),
Compare: CompareExactUnordered,
},
},
}
var varTests = TableTest{
Table: tbl(
"var_test",
srcHdrs(
srcHdr("_id", fldTypeID),
srcHdr("i1", fldTypeInt, "min 0", "max 1000"),
srcHdr("d1", fldTypeDecimal2),
srcHdr("s1", fldTypeString),
srcHdr("id1", fldTypeID),
),
srcRows(
srcRow(int64(1), int64(10), float64(10), string("foo"), int64(10)),
srcRow(int64(2), int64(10), float64(10), string("foo"), int64(11)),
srcRow(int64(3), int64(11), float64(11), string("foo"), int64(12)),
srcRow(int64(4), int64(12), float64(12), string("foo"), int64(13)),
srcRow(int64(5), int64(12), float64(12), string("foo"), int64(14)),
srcRow(int64(6), int64(13), float64(13), string("foo"), int64(15)),
),
),
SQLTests: []SQLTest{
{
SQLs: sqls(
"SELECT var(*) AS var1 FROM var_test",
),
ExpErr: "column reference expected",
},
{
SQLs: sqls(
"SELECT var(_id) AS var1 FROM var_test",
),
ExpErr: "_id column cannot be used in aggregate function 'var'",
},
{
SQLs: sqls(
"SELECT var(i1, d1) AS var1 FROM var_test",
),
ExpErr: "count of formal parameters (1) does not match count of actual parameters (2)",
},
{
SQLs: sqls(
"SELECT var(s1) AS var1 FROM var_test",
),
ExpErr: "integer, decimal or timestamp expression expected",
},
{
SQLs: sqls(
"SELECT var(id1) AS var1 FROM var_test",
),
ExpHdrs: hdrs(
hdr("var1", featurebase.WireQueryField{
Type: dax.BaseTypeDecimal + "(6)",
BaseType: dax.BaseTypeDecimal,
TypeInfo: map[string]interface{}{"scale": int64(6)},
}),
),
ExpRows: rows(
row(pql.NewDecimal(2916666, 6)),
),
Compare: CompareExactUnordered,
},
{
SQLs: sqls(
"SELECT var(i1) AS var1 FROM var_test",
"SELECT var(d1) AS var1 FROM var_test",
),
ExpHdrs: hdrs(
hdr("var1", featurebase.WireQueryField{
Type: dax.BaseTypeDecimal + "(6)",
BaseType: dax.BaseTypeDecimal,
TypeInfo: map[string]interface{}{"scale": int64(6)},
}),
),
ExpRows: rows(
row(pql.NewDecimal(1222222, 6)),
),
Compare: CompareExactUnordered,
},
{
SQLs: sqls(
"SELECT var(len(s1)) AS var1 FROM var_test",
),
ExpHdrs: hdrs(
hdr("var1", featurebase.WireQueryField{
Type: dax.BaseTypeDecimal + "(6)",
BaseType: dax.BaseTypeDecimal,
TypeInfo: map[string]interface{}{"scale": int64(6)},
}),
),
ExpRows: rows(
row(pql.NewDecimal(0, 6)),
),
Compare: CompareExactUnordered,
},
},
}

2
sql3/test/defs/defs_bulkinsert.go
View file

@ -1,6 +1,6 @@
package defs
// join tests
// bulk insert
var bulkInsertTable = TableTest{
name: "bulkInsertTable",
Table: tbl(

58
sql3/test/defs/defs_copy.go Normal file
View file

@ -0,0 +1,58 @@
// Copyright 2023 Molecula Corp. All rights reserved.
package defs
// copy
var copyTable = TableTest{
name: "copyTable",
Table: tbl(
"copytest",
srcHdrs(
srcHdr("_id", fldTypeID),
srcHdr("id_col", fldTypeID),
srcHdr("string_col", fldTypeString),
srcHdr("int_col", fldTypeInt),
srcHdr("decimal_col", fldTypeDecimal2),
srcHdr("bool_col", fldTypeBool),
srcHdr("time_col", fldTypeTimestamp),
srcHdr("stringset_col", fldTypeStringSet),
srcHdr("idset_col", fldTypeIDSet),
),
srcRows(
srcRow(int64(1), int64(10), string("foo"), int64(10), float64(10), bool(false), knownTimestamp(), []string{"foo", "bar"}, []int64{1, 2}),
srcRow(int64(2), int64(11), string("foo1"), int64(11), float64(11), bool(true), knownTimestamp(), []string{"foo1", "bar1"}, []int64{11, 21}),
srcRow(int64(3), int64(12), string("foo2"), int64(12), float64(12), bool(false), knownTimestamp(), []string{"foo2", "bar2"}, []int64{12, 22}),
srcRow(int64(4), int64(13), string("foo3"), int64(13), float64(13), bool(true), knownTimestamp(), []string{"foo3", "bar3"}, []int64{13, 23}),
),
),
SQLTests: nil,
}
var copyTests = TableTest{
name: "copyTests",
SQLTests: []SQLTest{
{
name: "copy-no-table-to-table",
SQLs: sqls(
`copy foo to bar;`,
),
ExpErr: "table or view 'foo' not found",
},
{
name: "copy-table-to-table-same-name",
SQLs: sqls(
`copy copytest to copytest;`,
),
ExpErr: "already exists",
},
{
name: "copy-table-to-table",
SQLs: sqls(
`copy copytest to copytesttwo;`,
),
ExpHdrs: hdrs(),
ExpRows: rows(),
Compare: CompareExactOrdered,
},
},
}

58
sql3/test/defs/defs_top.go
View file

@ -1,7 +1,7 @@
package defs
var topTests = TableTest{
name: "top-tests",
var topLimitTests = TableTest{
name: "top-limit-tests",
Table: tbl(
"skills",
srcHdrs(
@ -20,7 +20,25 @@ var topTests = TableTest{
SQLTests: []SQLTest{
{
SQLs: sqls(
"select top(1) * from skills where setcontains(skills, 'Marketing Manager');",
"select top(1) * from skills where setcontains(skills, 'Marketing Manager');",
),
ExpHdrs: hdrs(
hdr("_id", fldTypeID),
hdr("bools", fldTypeStringSet),
hdr("bools-exist", fldTypeStringSet),
hdr("id1", fldTypeID),
hdr("skills", fldTypeStringSet),
hdr("titles", fldTypeStringSet),
),
ExpRows: rows(
row(int64(1), nil, []string{"available_for_hire"}, int64(288), []string{"Marketing Manager"}, []string{"Alumni Relations", "OEM negotiations"}),
),
Compare: CompareExactUnordered,
SortStringKeys: true,
},
{
SQLs: sqls(
"select * from skills where setcontains(skills, 'Marketing Manager') limit 1;",
),
ExpHdrs: hdrs(
hdr("_id", fldTypeID),
@ -55,6 +73,21 @@ var topTests = TableTest{
Compare: CompareExactUnordered,
SortStringKeys: true,
},
{
SQLs: sqls(
"select count(*), skills from skills group by skills limit 10;",
),
ExpHdrs: hdrs(
hdr("", fldTypeInt),
hdr("skills", fldTypeStringSet),
),
ExpRows: rows(
row(int64(1), string("Marketing Manager")),
row(int64(1), string("Software Engineer I")),
),
Compare: CompareExactUnordered,
SortStringKeys: true,
},
{
SQLs: sqls(
"select top(1) count(*) from skills;",
@ -68,5 +101,24 @@ var topTests = TableTest{
Compare: CompareExactUnordered,
SortStringKeys: true,
},
{
SQLs: sqls(
"select count(*) from skills limit 1;",
),
ExpHdrs: hdrs(
hdr("", fldTypeInt),
),
ExpRows: rows(
row(int64(2)),
),
Compare: CompareExactUnordered,
SortStringKeys: true,
},
{
SQLs: sqls(
"select top(1) count(*) from skills limit 1;",
),
ExpErr: "TOP and LIMIT cannot cannot be used at the same time",
},
},
}