scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 09:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1910 from molecula/pql-variables

Browse source 
[FB-1063] Dynamically expand queries based on $variable values
This commit is contained in:
Samir Patel 2022-02-09 16:58:33 -05:00 • committed by GitHub
commit 1dc0d2c88b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 1560 additions and 1161 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

191
pql/ast.go
View file

@ -19,6 +19,22 @@ type Query struct {
conditional []string
}
// ExpandVars recursively replaces variables in the query with their values.
func (q *Query) ExpandVars(vars map[string]interface{}) (*Query, error) {
other := *q
other.Calls = make([]*Call, 0, len(q.Calls))
for _, c := range q.Calls {
newCalls, err := c.ExpandVars(vars)
if err != nil {
return nil, err
}
other.Calls = append(other.Calls, newCalls...)
}
return &other, nil
}
func (q *Query) startCall(name string) {
// Coerce every name into a canonical form if we know of one.
if canon, ok := canonicalCaps[strings.ToLower(name)]; ok {
@ -60,7 +76,11 @@ func (q *Query) addPosNum(key, value string) {
func (q *Query) addPosStr(key, value string) {
q.addField(key)
q.addVal(value)
if strings.HasPrefix(value, "$") {
q.addVal(NewVariable(strings.TrimPrefix(value, "$")))
} else {
q.addVal(value)
}
}
func (q *Query) startConditional() {
@ -351,11 +371,23 @@ type stringOrInt64Type struct{}
var stringOrInt64 stringOrInt64Type
// We want to be able to accept either a string or variable for
// _field args. Special-case type:
type stringOrVariableType struct{}
var stringOrVariable stringOrVariableType
// We want to be able to accept either a interface or variable for
// column args. Special-case type:
type interfaceOrVariableType struct{}
var interfaceOrVariable interfaceOrVariableType
var allowField = callInfo{
allowUnknown: false,
prototypes: map[string]interface{}{
"_field": "",
"field": "",
"_field": stringOrVariable,
"field": stringOrVariable,
},
}
@ -401,8 +433,8 @@ var callInfoByFunc = map[string]callInfo{
"Rows": {
allowUnknown: false,
prototypes: map[string]interface{}{
"_field": "",
"field": "",
"_field": stringOrVariable,
"field": stringOrVariable,
"limit": int64(0),
"column": nil,
"previous": nil,
@ -440,7 +472,7 @@ var callInfoByFunc = map[string]callInfo{
"ConstRow": {
allowUnknown: false,
prototypes: map[string]interface{}{
"columns": []interface{}{},
"columns": interfaceOrVariable,
},
callType: PrecallGlobal,
},
@ -448,8 +480,8 @@ var callInfoByFunc = map[string]callInfo{
"TopK": {
allowUnknown: false,
prototypes: map[string]interface{}{
"_field": "",
"field": "",
"_field": stringOrVariable,
"field": stringOrVariable,
"k": int64(0),
"filter": nil,
"from": nil,
@ -460,15 +492,15 @@ var callInfoByFunc = map[string]callInfo{
"TopN": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_field": "",
"field": "",
"_field": stringOrVariable,
"field": stringOrVariable,
},
},
"Percentile": {
allowUnknown: false,
prototypes: map[string]interface{}{
"field": "",
"_field": "",
"field": stringOrVariable,
"_field": stringOrVariable,
"filter": nil,
"nth": nil,
},
@ -577,6 +609,24 @@ func (c *Call) CheckCallInfo() error {
c.String(), k, v)
}
}
if reflect.TypeOf(acceptable) == reflect.TypeOf(stringOrVariable) {
switch v.(type) {
case string, *Variable:
continue
default:
return fmt.Errorf("'%s': arg '%s' needed a string or variable value, got %T",
c.String(), k, v)
}
}
if reflect.TypeOf(acceptable) == reflect.TypeOf(interfaceOrVariable) {
switch v.(type) {
case []interface{}, *Variable:
continue
default:
return fmt.Errorf("'%s': arg '%s' needed a []interface{} or variable value, got %T",
c.String(), k, v)
}
}
return fmt.Errorf("'%s': arg '%s' wrong type (got %T, expected %T)",
c.String(), k, v, acceptable)
}
@ -896,6 +946,106 @@ func (c *Call) ArgString(key string) string {
return s
}
// ExpandVars recursively replaces variables in the call with their values.
func (c *Call) ExpandVars(vars map[string]interface{}) ([]*Call, error) {
switch c.Name {
case "Row", "ConstRow", "Rows":
for argK, argV := range c.Args {
variable := getVariable(argV)
if variable == nil {
continue
}
for varK, varV := range vars {
if variable.Name != varK {
continue
}
switch values := varV.(type) {
case []interface{}:
return c.expandVars(argK, values), nil
default:
return nil, fmt.Errorf("expected variable value of type []interface{}, got: %T", values)
}
}
}
return []*Call{c}, nil
default:
other := *c
other.Args = CopyArgs(c.Args)
other.Children = make([]*Call, 0, len(c.Children))
for _, child := range c.Children {
newChildren, err := child.ExpandVars(vars)
if err != nil {
return nil, err
}
other.Children = append(other.Children, newChildren...)
}
for key, val := range other.Args {
switch call := val.(type) {
case *Call:
newArg, err := call.ExpandVars(vars)
if err != nil {
return nil, err
}
if len(newArg) != 1 {
return nil, fmt.Errorf("variable: requires single value for argument, got: %+v", newArg)
}
other.Args[key] = newArg[0]
}
}
return []*Call{&other}, nil
}
}
// expandVars specifies the implementation for variable expansion for various Call types
func (c *Call) expandVars(name string, values []interface{}) []*Call {
switch c.Name {
case "Row":
union := &Call{Name: "Union"}
for i := range values {
r := Call{Name: "Row", Args: CopyArgs(c.Args)}
switch cond := r.Args[name].(type) {
case *Condition:
r.Args[name] = &Condition{Op: cond.Op, Value: values[i]}
default:
r.Args[name] = values[i]
}
union.Children = append(union.Children, &r)
}
return []*Call{union}
case "Rows":
rows := make([]*Call, 0, len(values))
for i := range values {
r := Call{Name: "Rows"}
r.Args = CopyArgs(c.Args)
r.Args[name] = values[i]
rows = append(rows, &r)
}
return rows
case "ConstRow":
r := Call{Name: "ConstRow"}
r.Args = CopyArgs(c.Args)
r.Args[name] = values
return []*Call{&r}
}
return []*Call{c}
}
// getVariable returns *Variable given a Call argument if present
func getVariable(i interface{}) *Variable {
switch _var := i.(type) {
case *Condition:
if v, ok := _var.Value.(*Variable); ok {
return v
}
return nil
case *Variable: // if interface{} is of type Variable
return _var
default:
return nil
}
}
// Condition represents an operation & value.
// When used in an argument map it represents a binary expression.
type Condition struct {
@ -1034,6 +1184,21 @@ func (cond *Condition) StringSliceValue() ([]string, bool) {
return nil, false
}
// Variable represents a placeholder variable in a query.
type Variable struct {
Name string
}
// NewVariable returns a new instance of Variable.
func NewVariable(name string) *Variable {
return &Variable{Name: name}
}
// String returns the string representation of v.
func (v *Variable) String() string {
return "$" + v.Name
}
func formatValue(v interface{}) string {
switch v := v.(type) {
case nil:
@ -1048,6 +1213,8 @@ func formatValue(v interface{}) string {
return fmt.Sprintf("\"%s\"", v.Format(time.RFC3339Nano))
case *Condition:
return v.String()
case *Variable:
return v.String()
default:
return fmt.Sprintf("%v", v)
}

104
pql/ast_test.go
View file

@ -2,6 +2,7 @@
package pql_test
import (
"strings"
"testing"
"github.com/molecula/featurebase/v3/pql"
@ -52,3 +53,106 @@ func TestCondition_StringWithSubj(t *testing.T) {
}
}
}
func TestQuery_ExpandVars(t *testing.T) {
tests := []struct {
name string
input string
output string
vars map[string]interface{}
wantErr bool
}{
{
name: "ExpandRowEQInterior",
input: `count(row(animal=$var1))`,
output: `Count(Union(Row(animal="cat"), Row(animal="dog"), Row(animal="pig")))`,
vars: map[string]interface{}{"var1": []interface{}{"cat", "dog", "pig"}},
},
{
name: "ExpandRowEQExterior",
input: `row(animal=$var1)`,
output: `Union(Row(animal="cat"))`,
vars: map[string]interface{}{"var1": []interface{}{"cat"}},
},
{
name: "ExpandRowGT",
input: `count(row(num>$var1))`,
output: `Count(Union(Row(num>5), Row(num>10)))`,
vars: map[string]interface{}{"var1": []interface{}{5, 10}},
},
{
name: "ExpandRowNOT",
input: `count(row(num!=$var1))`,
output: `Count(Union(Row(num!=5), Row(num!=10)))`,
vars: map[string]interface{}{"var1": []interface{}{5, 10}},
},
{
name: "ExpandRowLTString",
input: `count(row(num<$var1))`,
output: `Count(Union(Row(num<"cat"), Row(num<"dog")))`,
vars: map[string]interface{}{"var1": []interface{}{"cat", "dog"}},
},
{
name: "ExpandRowsInterior",
input: `GroupBy(rows($var1), limit=5)`,
output: `GroupBy(Rows(_field="cat"), Rows(_field="dog"), limit=5)`,
vars: map[string]interface{}{"var1": []interface{}{"cat", "dog"}},
},
{
name: "ExpandRowsExterior",
input: `rows($var1)`,
output: `Rows(_field="cat")` + "\n" + `Rows(_field="dog")`,
vars: map[string]interface{}{"var1": []interface{}{"cat", "dog"}},
},
{
name: "ExpandRowAndRows",
input: `GroupBy(Rows($animal), limit=7, filter=Row(size=$size))`,
output: `GroupBy(Rows(_field="cat"), Rows(_field="dog"), filter=Union(Row(size="lg"), Row(size="md")), limit=7)`,
vars: map[string]interface{}{"animal": []interface{}{"cat", "dog"}, "size": []interface{}{"lg", "md"}},
},
{
name: "ExpandBad",
input: `$animal`,
vars: map[string]interface{}{"animal": []interface{}{"cat", "dog"}, "columns": []interface{}{5, 10}},
wantErr: true,
},
{
name: "ExpandBad2",
input: `GroupBy($animal)`,
output: `Intersect(ConstRow(columns=[5, 10]), Union(Row(animal="cat"), Row(animal="dog")))`,
wantErr: true,
},
{
name: "ExpandAsCSV",
input: `Intersect(ConstRow(columns=$var2), Row(animal=$var1))`,
output: `Intersect(ConstRow(columns=[5,10]), Union(Row(animal="cat"), Row(animal="dog")))`,
vars: map[string]interface{}{"var1": []interface{}{"cat", "dog"}, "var2": []interface{}{5, 10}},
},
{
name: "ExpandPercentile",
input: `Percentile(field="bytes", nth=99.0, filter=Row(level=$animal))`,
output: `Percentile(field="bytes", filter=Union(Row(level="cat"), Row(level="dog")), nth=99)`,
vars: map[string]interface{}{"animal": []interface{}{"cat", "dog"}, "columns": []interface{}{5, 10}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
q, err := pql.NewParser(strings.NewReader(tt.input)).Parse()
if err != nil {
if !tt.wantErr {
t.Errorf("Parse error = %v, wantErr %v", err, tt.wantErr)
}
return
}
got, err := q.ExpandVars(tt.vars)
if err != nil {
t.Errorf("Query.ExpandVars() error = %v", err)
return
}
if tt.output != got.String() {
t.Errorf("got %v, want %v", got, tt.output)
}
})
}
}

5
pql/pql.peg
View file

@ -43,6 +43,7 @@ items <- item (comma items)?
item <- 'null' &(comma / close) { p.addVal(nil) }
/ 'true' &(comma / close) { p.addVal(true) }
/ 'false' &(comma / close) { p.addVal(false) }
/ '$' < variable > { p.addVal(NewVariable(text)) }
/ timefmt { p.addVal(text) }
/ timestampfmt { p.addTimestampVal(text) }
/ < decimal > { p.addNumVal(text) }
@ -54,7 +55,9 @@ item <- 'null' &(comma / close) { p.addVal(nil) }
doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )*
singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )*
fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )*
variable <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )*
fieldExpr <- ( [[A-Z]] / '_' / '$' ) ( [[A-Z]] / [0-9] / '_' / '-' )*
field <- <fieldExpr / reserved> { p.addField(text) }
reserved <- '_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field'
posfield <- 'field='? <fieldExpr> { p.addPosStr("_field", text) }

2403
pql/pql.peg.go
View file

File diff suppressed because it is too large Load diff

18
pql/pqlpeg_test.go
View file

@ -752,6 +752,24 @@ func TestPQLDeepEquality(t *testing.T) {
{Name: "Rows"},
},
}},
{
name: "Variable",
call: "Row(f=$my_VAR123)",
exp: &Call{
Name: "Row",
Args: map[string]interface{}{
"f": &Variable{Name: "my_VAR123"},
},
}},
{
name: "RowsWithVariable",
call: `Rows($var)`,
exp: &Call{
Name: "Rows",
Args: map[string]interface{}{
"_field": &Variable{Name: "var"},
},
}},
}
for i, test := range tests {