Merge branch 'master' into cicd-will-it-never-end

.gitlab/.gitlab-ci.yml

@@ -379,10 +379,12 @@ gauntlet:
   timeout: 4h
   image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
   variables:
-    PROFILE: "service-terraform"
+    FBCI_PROFILE: "service-terraform"
+    INFRA_PROFILE: "service-gitlab"
     AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
     AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    ASG_NAME: "gitlab-runners"
     TF_VAR_cluster_prefix: ""
   tags:
     - aws
@@ -398,7 +400,9 @@ gauntlet:
     - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
     - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
     - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
+    - aws configure set aws_profile $FBCI_PROFILE
+    - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
+    - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
     - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
     - chmod 400 gitlab-featurebase-ci.pem
     - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
@@ -415,11 +419,17 @@ gauntlet:
     - export PATH=$PATH:/usr/local/go/bin
     - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
     - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+    - TF_VAR_branch=$CI_COMMIT_BRANCH
+    - echo "Branch --> $TF_VAR_branch"
+    - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
+    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
   script:
     - ./qa/scripts/setupSamsungGauntlet.sh
     - ./qa/scripts/testSamsungGauntlet.sh
   after_script:
-    - ./qa/scripts/teardownSamsungGauntlet.sh
+    - ./qa/scripts/teardownSamsungGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated
+    - export INSTANCE_ID=$(cat instance_id)
+    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
   
 s3 dump:
   stage: post build

pql/ast.go

@@ -19,6 +19,22 @@ type Query struct {
 	conditional []string
 }
 
+// ExpandVars recursively replaces variables in the query with their values.
+func (q *Query) ExpandVars(vars map[string]interface{}) (*Query, error) {
+	other := *q
+	other.Calls = make([]*Call, 0, len(q.Calls))
+
+	for _, c := range q.Calls {
+		newCalls, err := c.ExpandVars(vars)
+		if err != nil {
+			return nil, err
+		}
+		other.Calls = append(other.Calls, newCalls...)
+	}
+
+	return &other, nil
+}
+
 func (q *Query) startCall(name string) {
 	// Coerce every name into a canonical form if we know of one.
 	if canon, ok := canonicalCaps[strings.ToLower(name)]; ok {
@@ -60,7 +76,11 @@ func (q *Query) addPosNum(key, value string) {
 
 func (q *Query) addPosStr(key, value string) {
 	q.addField(key)
-	q.addVal(value)
+	if strings.HasPrefix(value, "$") {
+		q.addVal(NewVariable(strings.TrimPrefix(value, "$")))
+	} else {
+		q.addVal(value)
+	}
 }
 
 func (q *Query) startConditional() {
@@ -351,11 +371,23 @@ type stringOrInt64Type struct{}
 
 var stringOrInt64 stringOrInt64Type
 
+// We want to be able to accept either a string or variable for
+// _field args. Special-case type:
+type stringOrVariableType struct{}
+
+var stringOrVariable stringOrVariableType
+
+// We want to be able to accept either a interface or variable for
+// column args. Special-case type:
+type interfaceOrVariableType struct{}
+
+var interfaceOrVariable interfaceOrVariableType
+
 var allowField = callInfo{
 	allowUnknown: false,
 	prototypes: map[string]interface{}{
-		"_field": "",
-		"field":  "",
+		"_field": stringOrVariable,
+		"field":  stringOrVariable,
 	},
 }
 
@@ -401,8 +433,8 @@ var callInfoByFunc = map[string]callInfo{
 	"Rows": {
 		allowUnknown: false,
 		prototypes: map[string]interface{}{
-			"_field":   "",
-			"field":    "",
+			"_field":   stringOrVariable,
+			"field":    stringOrVariable,
 			"limit":    int64(0),
 			"column":   nil,
 			"previous": nil,
@@ -440,7 +472,7 @@ var callInfoByFunc = map[string]callInfo{
 	"ConstRow": {
 		allowUnknown: false,
 		prototypes: map[string]interface{}{
-			"columns": []interface{}{},
+			"columns": interfaceOrVariable,
 		},
 		callType: PrecallGlobal,
 	},
@@ -448,8 +480,8 @@ var callInfoByFunc = map[string]callInfo{
 	"TopK": {
 		allowUnknown: false,
 		prototypes: map[string]interface{}{
-			"_field": "",
-			"field":  "",
+			"_field": stringOrVariable,
+			"field":  stringOrVariable,
 			"k":      int64(0),
 			"filter": nil,
 			"from":   nil,
@@ -460,15 +492,15 @@ var callInfoByFunc = map[string]callInfo{
 	"TopN": {
 		allowUnknown: true,
 		prototypes: map[string]interface{}{
-			"_field": "",
-			"field":  "",
+			"_field": stringOrVariable,
+			"field":  stringOrVariable,
 		},
 	},
 	"Percentile": {
 		allowUnknown: false,
 		prototypes: map[string]interface{}{
-			"field":  "",
-			"_field": "",
+			"field":  stringOrVariable,
+			"_field": stringOrVariable,
 			"filter": nil,
 			"nth":    nil,
 		},
@@ -577,6 +609,24 @@ func (c *Call) CheckCallInfo() error {
 					c.String(), k, v)
 			}
 		}
+		if reflect.TypeOf(acceptable) == reflect.TypeOf(stringOrVariable) {
+			switch v.(type) {
+			case string, *Variable:
+				continue
+			default:
+				return fmt.Errorf("'%s': arg '%s' needed a string or variable value, got %T",
+					c.String(), k, v)
+			}
+		}
+		if reflect.TypeOf(acceptable) == reflect.TypeOf(interfaceOrVariable) {
+			switch v.(type) {
+			case []interface{}, *Variable:
+				continue
+			default:
+				return fmt.Errorf("'%s': arg '%s' needed a []interface{} or variable value, got %T",
+					c.String(), k, v)
+			}
+		}
 		return fmt.Errorf("'%s': arg '%s' wrong type (got %T, expected %T)",
 			c.String(), k, v, acceptable)
 	}
@@ -896,6 +946,106 @@ func (c *Call) ArgString(key string) string {
 	return s
 }
 
+// ExpandVars recursively replaces variables in the call with their values.
+func (c *Call) ExpandVars(vars map[string]interface{}) ([]*Call, error) {
+	switch c.Name {
+	case "Row", "ConstRow", "Rows":
+		for argK, argV := range c.Args {
+			variable := getVariable(argV)
+			if variable == nil {
+				continue
+			}
+			for varK, varV := range vars {
+				if variable.Name != varK {
+					continue
+				}
+				switch values := varV.(type) {
+				case []interface{}:
+					return c.expandVars(argK, values), nil
+				default:
+					return nil, fmt.Errorf("expected variable value of type []interface{}, got: %T", values)
+				}
+
+			}
+		}
+		return []*Call{c}, nil
+	default:
+		other := *c
+		other.Args = CopyArgs(c.Args)
+		other.Children = make([]*Call, 0, len(c.Children))
+		for _, child := range c.Children {
+			newChildren, err := child.ExpandVars(vars)
+			if err != nil {
+				return nil, err
+			}
+			other.Children = append(other.Children, newChildren...)
+		}
+		for key, val := range other.Args {
+			switch call := val.(type) {
+			case *Call:
+				newArg, err := call.ExpandVars(vars)
+				if err != nil {
+					return nil, err
+				}
+				if len(newArg) != 1 {
+					return nil, fmt.Errorf("variable: requires single value for argument, got: %+v", newArg)
+				}
+				other.Args[key] = newArg[0]
+			}
+		}
+		return []*Call{&other}, nil
+	}
+}
+
+// expandVars specifies the implementation for variable expansion for various Call types
+func (c *Call) expandVars(name string, values []interface{}) []*Call {
+	switch c.Name {
+	case "Row":
+		union := &Call{Name: "Union"}
+		for i := range values {
+			r := Call{Name: "Row", Args: CopyArgs(c.Args)}
+			switch cond := r.Args[name].(type) {
+			case *Condition:
+				r.Args[name] = &Condition{Op: cond.Op, Value: values[i]}
+			default:
+				r.Args[name] = values[i]
+			}
+			union.Children = append(union.Children, &r)
+		}
+		return []*Call{union}
+	case "Rows":
+		rows := make([]*Call, 0, len(values))
+		for i := range values {
+			r := Call{Name: "Rows"}
+			r.Args = CopyArgs(c.Args)
+			r.Args[name] = values[i]
+			rows = append(rows, &r)
+		}
+		return rows
+	case "ConstRow":
+		r := Call{Name: "ConstRow"}
+		r.Args = CopyArgs(c.Args)
+		r.Args[name] = values
+		return []*Call{&r}
+	}
+	return []*Call{c}
+}
+
+// getVariable returns *Variable given a Call argument if present
+func getVariable(i interface{}) *Variable {
+	switch _var := i.(type) {
+	case *Condition:
+		if v, ok := _var.Value.(*Variable); ok {
+			return v
+		}
+		return nil
+	case *Variable: // if interface{} is of type Variable
+		return _var
+	default:
+		return nil
+	}
+}
+
 // Condition represents an operation & value.
 // When used in an argument map it represents a binary expression.
 type Condition struct {
@@ -1034,6 +1184,21 @@ func (cond *Condition) StringSliceValue() ([]string, bool) {
 	return nil, false
 }
 
+// Variable represents a placeholder variable in a query.
+type Variable struct {
+	Name string
+}
+
+// NewVariable returns a new instance of Variable.
+func NewVariable(name string) *Variable {
+	return &Variable{Name: name}
+}
+
+// String returns the string representation of v.
+func (v *Variable) String() string {
+	return "$" + v.Name
+}
+
 func formatValue(v interface{}) string {
 	switch v := v.(type) {
 	case nil:
@@ -1048,6 +1213,8 @@ func formatValue(v interface{}) string {
 		return fmt.Sprintf("\"%s\"", v.Format(time.RFC3339Nano))
 	case *Condition:
 		return v.String()
+	case *Variable:
+		return v.String()
 	default:
 		return fmt.Sprintf("%v", v)
 	}

pql/ast_test.go

@@ -2,6 +2,7 @@
 package pql_test
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/molecula/featurebase/v3/pql"
@@ -52,3 +53,106 @@ func TestCondition_StringWithSubj(t *testing.T) {
 		}
 	}
 }
+
+func TestQuery_ExpandVars(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   string
+		output  string
+		vars    map[string]interface{}
+		wantErr bool
+	}{
+		{
+			name:   "ExpandRowEQInterior",
+			input:  `count(row(animal=$var1))`,
+			output: `Count(Union(Row(animal="cat"), Row(animal="dog"), Row(animal="pig")))`,
+			vars:   map[string]interface{}{"var1": []interface{}{"cat", "dog", "pig"}},
+		},
+		{
+			name:   "ExpandRowEQExterior",
+			input:  `row(animal=$var1)`,
+			output: `Union(Row(animal="cat"))`,
+			vars:   map[string]interface{}{"var1": []interface{}{"cat"}},
+		},
+		{
+			name:   "ExpandRowGT",
+			input:  `count(row(num>$var1))`,
+			output: `Count(Union(Row(num>5), Row(num>10)))`,
+			vars:   map[string]interface{}{"var1": []interface{}{5, 10}},
+		},
+		{
+			name:   "ExpandRowNOT",
+			input:  `count(row(num!=$var1))`,
+			output: `Count(Union(Row(num!=5), Row(num!=10)))`,
+			vars:   map[string]interface{}{"var1": []interface{}{5, 10}},
+		},
+		{
+			name:   "ExpandRowLTString",
+			input:  `count(row(num<$var1))`,
+			output: `Count(Union(Row(num<"cat"), Row(num<"dog")))`,
+			vars:   map[string]interface{}{"var1": []interface{}{"cat", "dog"}},
+		},
+		{
+			name:   "ExpandRowsInterior",
+			input:  `GroupBy(rows($var1), limit=5)`,
+			output: `GroupBy(Rows(_field="cat"), Rows(_field="dog"), limit=5)`,
+			vars:   map[string]interface{}{"var1": []interface{}{"cat", "dog"}},
+		},
+		{
+			name:   "ExpandRowsExterior",
+			input:  `rows($var1)`,
+			output: `Rows(_field="cat")` + "\n" + `Rows(_field="dog")`,
+			vars:   map[string]interface{}{"var1": []interface{}{"cat", "dog"}},
+		},
+		{
+			name:   "ExpandRowAndRows",
+			input:  `GroupBy(Rows($animal), limit=7, filter=Row(size=$size))`,
+			output: `GroupBy(Rows(_field="cat"), Rows(_field="dog"), filter=Union(Row(size="lg"), Row(size="md")), limit=7)`,
+			vars:   map[string]interface{}{"animal": []interface{}{"cat", "dog"}, "size": []interface{}{"lg", "md"}},
+		},
+		{
+			name:    "ExpandBad",
+			input:   `$animal`,
+			vars:    map[string]interface{}{"animal": []interface{}{"cat", "dog"}, "columns": []interface{}{5, 10}},
+			wantErr: true,
+		},
+		{
+			name:    "ExpandBad2",
+			input:   `GroupBy($animal)`,
+			output:  `Intersect(ConstRow(columns=[5, 10]), Union(Row(animal="cat"), Row(animal="dog")))`,
+			wantErr: true,
+		},
+		{
+			name:   "ExpandAsCSV",
+			input:  `Intersect(ConstRow(columns=$var2), Row(animal=$var1))`,
+			output: `Intersect(ConstRow(columns=[5,10]), Union(Row(animal="cat"), Row(animal="dog")))`,
+			vars:   map[string]interface{}{"var1": []interface{}{"cat", "dog"}, "var2": []interface{}{5, 10}},
+		},
+		{
+			name:   "ExpandPercentile",
+			input:  `Percentile(field="bytes", nth=99.0, filter=Row(level=$animal))`,
+			output: `Percentile(field="bytes", filter=Union(Row(level="cat"), Row(level="dog")), nth=99)`,
+			vars:   map[string]interface{}{"animal": []interface{}{"cat", "dog"}, "columns": []interface{}{5, 10}},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			q, err := pql.NewParser(strings.NewReader(tt.input)).Parse()
+			if err != nil {
+				if !tt.wantErr {
+					t.Errorf("Parse error = %v, wantErr %v", err, tt.wantErr)
+				}
+				return
+			}
+
+			got, err := q.ExpandVars(tt.vars)
+			if err != nil {
+				t.Errorf("Query.ExpandVars() error = %v", err)
+				return
+			}
+			if tt.output != got.String() {
+				t.Errorf("got %v, want %v", got, tt.output)
+			}
+		})
+	}
+}

pql/pql.peg

@@ -43,6 +43,7 @@ items <- item (comma items)?
 item <-   'null' &(comma / close) { p.addVal(nil) }
          / 'true' &(comma / close) { p.addVal(true) }
          / 'false' &(comma / close) { p.addVal(false) }
+         / '$' < variable > { p.addVal(NewVariable(text)) }
          / timefmt { p.addVal(text) }
          / timestampfmt { p.addTimestampVal(text) }
          / < decimal > { p.addNumVal(text) }
@@ -54,7 +55,9 @@ item <-   'null' &(comma / close) { p.addVal(nil) }
 doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )*
 singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' /  [^'\\] )*
 
-fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )*
+variable <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )*
+
+fieldExpr <- ( [[A-Z]] / '_' / '$' ) ( [[A-Z]] / [0-9] / '_' / '-' )*
 field <- <fieldExpr / reserved> { p.addField(text) }
 reserved <- '_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field'
 posfield <- 'field='? <fieldExpr> { p.addPosStr("_field", text) }

pql/pqlpeg_test.go

@@ -752,6 +752,24 @@ func TestPQLDeepEquality(t *testing.T) {
 					{Name: "Rows"},
 				},
 			}},
+		{
+			name: "Variable",
+			call: "Row(f=$my_VAR123)",
+			exp: &Call{
+				Name: "Row",
+				Args: map[string]interface{}{
+					"f": &Variable{Name: "my_VAR123"},
+				},
+			}},
+		{
+			name: "RowsWithVariable",
+			call: `Rows($var)`,
+			exp: &Call{
+				Name: "Rows",
+				Args: map[string]interface{}{
+					"_field": &Variable{Name: "var"},
+				},
+			}},
 	}
 
 	for i, test := range tests {