From 7cf2c5b07ed6689fe99a07f8fc2364d3199c457a Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 31 Mar 2023 11:04:25 -0500 Subject: [PATCH] handle integers as comparisons for decimal fields It's reasonable to allow "where x > 13" on decimal fields. Handle at least int64 and float64. Once this is up, we find that aggregates can return non-values, such as nil, in some cases; for instance, `percentile(x) where x > 13` can yield a nil if x is never greater than 13, rather than making up a value from zero data points. So we accept nil as a valid result type in PQL aggregates. As a result of this, change two tests which were unintentionally testing for an arcane edge case bug in which (1) we can't render a condition to PQL, such as because you specified an integer for a decimal field, and (2) the filter is using an aliased name, in which we would end up failing to generate a PQL filter, but *also* losing the SQL-layer filter, and produce wrong results as though there were no filter. We also alter the tests to use `o.price > 9`, because this lets us generate three user names, but only two distinct user names, so the test using DISTINCT returns a different value than the test not using DISTINCT, which helps us verify that it's actually working and not just lucky. As part of fixing that, there was an intermediate state where we rejected as an error any case where generating the PQL filter failed. This broke 21 more test cases, but in all of those cases, the SQL filter was actually working. ... But in two of them, we SHOULD have been able to generate PQL, because they were testing bools for null, which works fine. We just had a list of field types we allowed null tests against and omitted bool because I forgot that bool isn't always just treated as a kind of mutex. --- sql3/planner/expressionpql.go | 17 +++++++++-------- sql3/planner/oppqlaggregate.go | 4 ++++ sql3/planner/planoptimizer.go | 2 +- sql3/test/defs/defs_join.go | 8 ++++---- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/sql3/planner/expressionpql.go b/sql3/planner/expressionpql.go index 48526a841..7d19e3ac7 100644 --- a/sql3/planner/expressionpql.go +++ b/sql3/planner/expressionpql.go @@ -499,18 +499,19 @@ func (p *ExecutionPlanner) generatePQLCallFromBinaryExpr(ctx context.Context, ex if err != nil { return nil, err } - val, ok := pqlValue.(float64) - if !ok { + cond := &pql.Condition{Op: pqlOp} + switch val := pqlValue.(type) { + case float64: + cond.Value = pql.FromFloat64(val) + case int64: + cond.Value = pql.FromInt64(val, 0) + default: return nil, sql3.NewErrInternalf("unexpected type '%T", pqlValue) } - d := pql.FromFloat64(val) return &pql.Call{ Name: "Row", Args: map[string]interface{}{ - lhs.columnName: &pql.Condition{ - Op: pqlOp, - Value: d, - }, + lhs.columnName: cond, }, }, nil @@ -541,7 +542,7 @@ func (p *ExecutionPlanner) generatePQLCallFromBinaryExpr(ctx context.Context, ex pqlOp = pql.NEQ } switch typ := expr.lhs.Type().(type) { - case *parser.DataTypeID, *parser.DataTypeString, *parser.DataTypeIDSet, *parser.DataTypeStringSet: + case *parser.DataTypeID, *parser.DataTypeString, *parser.DataTypeIDSet, *parser.DataTypeStringSet, *parser.DataTypeBool: if strings.EqualFold(lhs.columnName, string(dax.PrimaryKeyFieldName)) { return nil, sql3.NewErrInvalidColumnInFilterExpression(0, 0, string(dax.PrimaryKeyFieldName), "is/is not null") } diff --git a/sql3/planner/oppqlaggregate.go b/sql3/planner/oppqlaggregate.go index b8938d4bf..a4c43ba7a 100644 --- a/sql3/planner/oppqlaggregate.go +++ b/sql3/planner/oppqlaggregate.go @@ -294,6 +294,10 @@ func (i *pqlAggregateRowIter) Next(ctx context.Context) (types.Row, error) { default: return nil, sql3.NewErrInternalf("unhandled return type '%T'", i.aggregate.Type()) } + case nil: + // it's valid for an aggregate to yield a NULL in some cases, such as + // when it's called on what turns out to be an empty set. + i.resultValue = nil default: return nil, sql3.NewErrInternalf("unexpected result type '%T'", queryResponse.Results[0]) } diff --git a/sql3/planner/planoptimizer.go b/sql3/planner/planoptimizer.go index c49813b15..fe73539eb 100644 --- a/sql3/planner/planoptimizer.go +++ b/sql3/planner/planoptimizer.go @@ -102,7 +102,7 @@ func (p *ExecutionPlanner) optimizePlan(ctx context.Context, plan types.PlanOper dumpPlan( []string{"================================================================================", "plan post-optimzation"}, - plan, + result, "--------------------------------------------------------------------------------", ) diff --git a/sql3/test/defs/defs_join.go b/sql3/test/defs/defs_join.go index 8ee73afd1..e52831d51 100644 --- a/sql3/test/defs/defs_join.go +++ b/sql3/test/defs/defs_join.go @@ -113,26 +113,26 @@ var joinTests = TableTest{ { name: "innerjoin-aggregate-groupby-count-distinct-filter", SQLs: sqls( - "SELECT COUNT(DISTINCT u.name) FROM orders o JOIN users u ON o.userid = u._id WHERE o.price > 10;", + "SELECT COUNT(DISTINCT u.name) FROM orders o JOIN users u ON o.userid = u._id WHERE o.price > 9;", ), ExpHdrs: hdrs( hdr("", fldTypeInt), ), ExpRows: rows( - row(int64(4)), + row(int64(2)), ), Compare: CompareExactOrdered, }, { name: "innerjoin-aggregate-groupby-count-filter", SQLs: sqls( - "SELECT COUNT(u.name) FROM orders o JOIN users u ON o.userid = u._id WHERE o.price > 10;", + "SELECT COUNT(u.name) FROM orders o JOIN users u ON o.userid = u._id WHERE o.price > 9;", ), ExpHdrs: hdrs( hdr("", fldTypeInt), ), ExpRows: rows( - row(int64(6)), + row(int64(3)), ), Compare: CompareExactOrdered, },