From 40d292b5899935db9b8ccb1d337cfaa997c02ef7 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula <85502298+pokeeffe-molecula@users.noreply.github.com> Date: Thu, 8 Dec 2022 15:41:11 -0600 Subject: [PATCH] handle int-->bool map type conversions; handle single value-->(id|string)set map type conversions (#2342) (cherry picked from commit 728b1dc9f5c00fc7ce74796c0141add3b89d7f13) --- sql3/errors.go | 16 +++++++++++++ sql3/planner/opbulkinsert.go | 22 +++++++++++++----- sql3/sql_complex_test.go | 45 +++++++++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/sql3/errors.go b/sql3/errors.go index 7eca5d0bc..8efd7ef47 100644 --- a/sql3/errors.go +++ b/sql3/errors.go @@ -108,6 +108,8 @@ const ( ErrInvalidInputSpecifier errors.Code = "ErrInvalidInputSpecifier" ErrInvalidBatchSize errors.Code = "ErrInvalidBatchSize" ErrTypeConversionOnMap errors.Code = "ErrTypeConversionOnMap" + ErrParsingJSON errors.Code = "ErrParsingJSON" + ErrEvaluatingJSONPathExpr errors.Code = "ErrEvaluatingJSONPathExpr" // optimizer errors ErrAggregateNotAllowedInGroupBy errors.Code = "ErrIdPercentileNotAllowedInGroupBy" @@ -663,6 +665,20 @@ func NewErrTypeConversionOnMap(line, col int, value interface{}, typeName string ) } +func NewErrParsingJSON(line, col int, jsonString string, errorText string) error { + return errors.New( + ErrParsingJSON, + fmt.Sprintf("[%d:%d] unable to parse JSON '%s': %s", line, col, jsonString, errorText), + ) +} + +func NewErrEvaluatingJSONPathExpr(line, col int, exprText string, jsonString string, errorText string) error { + return errors.New( + ErrEvaluatingJSONPathExpr, + fmt.Sprintf("[%d:%d] unable to evaluate JSONPath expression '%s' in '%s': %s", line, col, exprText, jsonString, errorText), + ) +} + // optimizer func NewErrAggregateNotAllowedInGroupBy(line, col int, aggName string) error { diff --git a/sql3/planner/opbulkinsert.go b/sql3/planner/opbulkinsert.go index 0d5f7b2f1..0177ec615 100644 --- a/sql3/planner/opbulkinsert.go +++ b/sql3/planner/opbulkinsert.go @@ -303,7 +303,7 @@ func (i *bulkInsertSourceCSVRowIter) Next(ctx context.Context) (types.Row, error result[idx] = evalValue case *parser.DataTypeBool: - bval, err := strconv.ParseInt(evalValue, 10, 64) + bval, err := strconv.ParseBool(evalValue) if err != nil { return nil, sql3.NewErrTypeConversionOnMap(0, 0, evalValue, mapColumn.colType.TypeDescription()) } @@ -472,7 +472,7 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er err := json.Unmarshal([]byte(jsonValue), &v) if err != nil { - return nil, err + return nil, sql3.NewErrParsingJSON(0, 0, jsonValue, err.Error()) } // type check against the output type of the map operation @@ -481,7 +481,7 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er evalValue, err := expr(ctx, v) if err != nil { - return nil, err + return nil, sql3.NewErrEvaluatingJSONPathExpr(0, 0, i.mapExpressionResults[idx], jsonValue, err.Error()) } // if nil (null) then return nil @@ -526,7 +526,12 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er case *parser.DataTypeIDSet: switch v := evalValue.(type) { case float64: - return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription()) + // if v is a whole number then make it an int, and then turn that into an idset + if v == float64(int64(v)) { + result[idx] = []int64{int64(v)} + } else { + return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription()) + } case []interface{}: setValue := make([]int64, 0) @@ -573,7 +578,7 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er result[idx] = setValue case string: - return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription()) + result[idx] = []string{v} case bool: return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription()) @@ -648,7 +653,12 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er case *parser.DataTypeBool: switch v := evalValue.(type) { case float64: - return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription()) + // if a whole number make it an int, and convert to a bool + if v == float64(int64(v)) { + result[idx] = v > 0 + } else { + return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription()) + } case []interface{}: return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription()) diff --git a/sql3/sql_complex_test.go b/sql3/sql_complex_test.go index 2b39393da..a33c9d508 100644 --- a/sql3/sql_complex_test.go +++ b/sql3/sql_complex_test.go @@ -1680,7 +1680,10 @@ func TestPlanner_BulkInsert(t *testing.T) { map ('$._id' id, '$.id1' id, '$.i1' int, '$.ids1' idset, '$.ss1' stringset, '$.ts1' timestamp, '$.s1' string, '$.b1' bool, '$.d1' decimal(2)) from - x'{ "_id": 1, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": true, "d1": 11.34 }' + x'{ "_id": 1, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": true, "d1": 11.34 } + { "_id": 2, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 0, "d1": 11.34 } + { "_id": 3, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 } + { "_id": 4, "id1": 10, "i1": 11, "ids1": 9, "ss1": "baz", "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 }' with format 'NDJSON' input 'STREAM';`) @@ -1689,6 +1692,46 @@ func TestPlanner_BulkInsert(t *testing.T) { } }) + t.Run("BulkNDJsonBadJsonPath", func(t *testing.T) { + + _, _, err = sql_test.MustQueryRows(t, c.GetNode(0).Server, `bulk insert + into alltypes (_id, id1, i1, ids1, ss1, ts1, s1, b1, d1) + + map ('$._id' id, '$.id1' id, '$.i1' int, '$.ids1' idset, '$.ss1' stringset, '$.ts1' timestamp, '$.s1' string, '$.blah' bool, '$.d1' decimal(2)) + + from + x'{ "_id": 1, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": true, "d1": 11.34 } + { "_id": 2, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 0, "d1": 11.34 } + { "_id": 3, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 } + { "_id": 4, "id1": 10, "i1": 11, "ids1": 9, "ss1": "baz", "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 }' + with + format 'NDJSON' + input 'STREAM';`) + if err == nil || !strings.Contains(err.Error(), `unknown key blah`) { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("BulkNDJsonBadJson", func(t *testing.T) { + + _, _, err = sql_test.MustQueryRows(t, c.GetNode(0).Server, `bulk insert + into alltypes (_id, id1, i1, ids1, ss1, ts1, s1, b1, d1) + + map ('$._id' id, '$.id1' id, '$.i1' int, '$.ids1' idset, '$.ss1' stringset, '$.ts1' timestamp, '$.s1' string, '$.b1' bool, '$.d1' decimal(2)) + + from + x'{ "_id": 1, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny" "b1": true, "d1": 11.34 } + { "_id": 2, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 0, "d1": 11.34 } + { "_id": 3, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 } + { "_id": 4, "id1": 10, "i1": 11, "ids1": 9, "ss1": "baz", "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 }' + with + format 'NDJSON' + input 'STREAM';`) + if err == nil || !strings.Contains(err.Error(), `: invalid character '"' after object key:value pair`) { + t.Fatalf("unexpected error: %v", err) + } + }) + t.Run("BulkInsertDecimals", func(t *testing.T) { _, _, err = sql_test.MustQueryRows(t, c.GetNode(0).Server, `create table iris (