scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-15 08:41:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

handle int-->bool map type conversions; handle single value-->(id|string)set map type conversions (#2342)

Browse source 
(cherry picked from commit 728b1dc9f5)
This commit is contained in:
pokeeffe-molecula 2022-12-08 15:41:11 -06:00 • committed by Fletcher Haynes
parent bf0486503a
commit 40d292b589
3 changed files with 76 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
sql3/errors.go
View file

@ -108,6 +108,8 @@ const (
ErrInvalidInputSpecifier errors.Code = "ErrInvalidInputSpecifier"
ErrInvalidBatchSize errors.Code = "ErrInvalidBatchSize"
ErrTypeConversionOnMap errors.Code = "ErrTypeConversionOnMap"
ErrParsingJSON errors.Code = "ErrParsingJSON"
ErrEvaluatingJSONPathExpr errors.Code = "ErrEvaluatingJSONPathExpr"
// optimizer errors
ErrAggregateNotAllowedInGroupBy errors.Code = "ErrIdPercentileNotAllowedInGroupBy"
@ -663,6 +665,20 @@ func NewErrTypeConversionOnMap(line, col int, value interface{}, typeName string
)
}
func NewErrParsingJSON(line, col int, jsonString string, errorText string) error {
return errors.New(
ErrParsingJSON,
fmt.Sprintf("[%d:%d] unable to parse JSON '%s': %s", line, col, jsonString, errorText),
)
}
func NewErrEvaluatingJSONPathExpr(line, col int, exprText string, jsonString string, errorText string) error {
return errors.New(
ErrEvaluatingJSONPathExpr,
fmt.Sprintf("[%d:%d] unable to evaluate JSONPath expression '%s' in '%s': %s", line, col, exprText, jsonString, errorText),
)
}
// optimizer
func NewErrAggregateNotAllowedInGroupBy(line, col int, aggName string) error {

22
sql3/planner/opbulkinsert.go
View file

@ -303,7 +303,7 @@ func (i *bulkInsertSourceCSVRowIter) Next(ctx context.Context) (types.Row, error
result[idx] = evalValue
case *parser.DataTypeBool:
bval, err := strconv.ParseInt(evalValue, 10, 64)
bval, err := strconv.ParseBool(evalValue)
if err != nil {
return nil, sql3.NewErrTypeConversionOnMap(0, 0, evalValue, mapColumn.colType.TypeDescription())
}
@ -472,7 +472,7 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er
err := json.Unmarshal([]byte(jsonValue), &v)
if err != nil {
return nil, err
return nil, sql3.NewErrParsingJSON(0, 0, jsonValue, err.Error())
}
// type check against the output type of the map operation
@ -481,7 +481,7 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er
evalValue, err := expr(ctx, v)
if err != nil {
return nil, err
return nil, sql3.NewErrEvaluatingJSONPathExpr(0, 0, i.mapExpressionResults[idx], jsonValue, err.Error())
}
// if nil (null) then return nil
@ -526,7 +526,12 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er
case *parser.DataTypeIDSet:
switch v := evalValue.(type) {
case float64:
return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription())
// if v is a whole number then make it an int, and then turn that into an idset
if v == float64(int64(v)) {
result[idx] = []int64{int64(v)}
} else {
return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription())
}
case []interface{}:
setValue := make([]int64, 0)
@ -573,7 +578,7 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er
result[idx] = setValue
case string:
return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription())
result[idx] = []string{v}
case bool:
return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription())
@ -648,7 +653,12 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er
case *parser.DataTypeBool:
switch v := evalValue.(type) {
case float64:
return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription())
// if a whole number make it an int, and convert to a bool
if v == float64(int64(v)) {
result[idx] = v > 0
} else {
return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription())
}
case []interface{}:
return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription())

45
sql3/sql_complex_test.go
View file

@ -1680,7 +1680,10 @@ func TestPlanner_BulkInsert(t *testing.T) {
map ('$._id' id, '$.id1' id, '$.i1' int, '$.ids1' idset, '$.ss1' stringset, '$.ts1' timestamp, '$.s1' string, '$.b1' bool, '$.d1' decimal(2))
from
x'{ "_id": 1, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": true, "d1": 11.34 }'
x'{ "_id": 1, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": true, "d1": 11.34 }
{ "_id": 2, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 0, "d1": 11.34 }
{ "_id": 3, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 }
{ "_id": 4, "id1": 10, "i1": 11, "ids1": 9, "ss1": "baz", "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 }'
with
format 'NDJSON'
input 'STREAM';`)
@ -1689,6 +1692,46 @@ func TestPlanner_BulkInsert(t *testing.T) {
}
})
t.Run("BulkNDJsonBadJsonPath", func(t *testing.T) {
_, _, err = sql_test.MustQueryRows(t, c.GetNode(0).Server, `bulk insert
into alltypes (_id, id1, i1, ids1, ss1, ts1, s1, b1, d1)
map ('$._id' id, '$.id1' id, '$.i1' int, '$.ids1' idset, '$.ss1' stringset, '$.ts1' timestamp, '$.s1' string, '$.blah' bool, '$.d1' decimal(2))
from
x'{ "_id": 1, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": true, "d1": 11.34 }
{ "_id": 2, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 0, "d1": 11.34 }
{ "_id": 3, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 }
{ "_id": 4, "id1": 10, "i1": 11, "ids1": 9, "ss1": "baz", "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 }'
with
format 'NDJSON'
input 'STREAM';`)
if err == nil || !strings.Contains(err.Error(), `unknown key blah`) {
t.Fatalf("unexpected error: %v", err)
}
})
t.Run("BulkNDJsonBadJson", func(t *testing.T) {
_, _, err = sql_test.MustQueryRows(t, c.GetNode(0).Server, `bulk insert
into alltypes (_id, id1, i1, ids1, ss1, ts1, s1, b1, d1)
map ('$._id' id, '$.id1' id, '$.i1' int, '$.ids1' idset, '$.ss1' stringset, '$.ts1' timestamp, '$.s1' string, '$.b1' bool, '$.d1' decimal(2))
from
x'{ "_id": 1, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny" "b1": true, "d1": 11.34 }
{ "_id": 2, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 0, "d1": 11.34 }
{ "_id": 3, "id1": 10, "i1": 11, "ids1": [ 3, 4, 5 ], "ss1": [ "foo", "bar" ], "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 }
{ "_id": 4, "id1": 10, "i1": 11, "ids1": 9, "ss1": "baz", "ts1": "2012-11-01T22:08:41+00:00", "s1": "frobny", "b1": 1, "d1": 11.34 }'
with
format 'NDJSON'
input 'STREAM';`)
if err == nil || !strings.Contains(err.Error(), `: invalid character '"' after object key:value pair`) {
t.Fatalf("unexpected error: %v", err)
}
})
t.Run("BulkInsertDecimals", func(t *testing.T) {
_, _, err = sql_test.MustQueryRows(t, c.GetNode(0).Server, `create table iris (