FB-1968 timestamp data type related fixes and enhancements (#2256)

* Removed support for EPOCH column constraint from TIMESTAMP SQL data type.
* Implicit conversion of integers to timestamp will treat the integer value as seconds since unix epoch.
* Add new ToTimeStamp(num, timeunit) SQL scalar function to help convert integer values to timestamp.

dax/table.go

@@ -755,9 +755,6 @@ func (f *Field) constraints() string {
 	case BaseTypeTimestamp:
 		if f.Options.TimeUnit != "" {
 			sql += fmt.Sprintf(" TIMEUNIT '%s'", f.Options.TimeUnit)
-			if !f.Options.Epoch.IsZero() {
-				sql += fmt.Sprintf(" EPOCH '%s'", f.Options.Epoch.Format(time.RFC3339)) // time.RFC3339
-			}
 		}
 	}
 

dax/table_test.go

@@ -6,7 +6,6 @@ import (
 	"sort"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/pql"
@@ -193,12 +192,11 @@ func TestTable(t *testing.T) {
 							Type: "timestamp",
 							Options: dax.FieldOptions{
 								TimeUnit: "s",
-								Epoch:    time.Date(2009, 11, 10, 23, 34, 56, 0, time.UTC),
 							},
 						},
 					},
 				},
-				expSQL: "CREATE TABLE all_field_types_with_options (_id string, an_id id CACHETYPE ranked SIZE 500, a_string string CACHETYPE ranked SIZE 500, an_id_set idset CACHETYPE ranked SIZE 500, a_string_set stringset CACHETYPE ranked SIZE 500, an_int int MIN -100 MAX 200, a_decimal decimal, a_timestamp timestamp TIMEUNIT 's' EPOCH '2009-11-10T23:34:56Z') KEYPARTITIONS 0",
+				expSQL: "CREATE TABLE all_field_types_with_options (_id string, an_id id CACHETYPE ranked SIZE 500, a_string string CACHETYPE ranked SIZE 500, an_id_set idset CACHETYPE ranked SIZE 500, a_string_set stringset CACHETYPE ranked SIZE 500, an_int int MIN -100 MAX 200, a_decimal decimal, a_timestamp timestamp TIMEUNIT 's') KEYPARTITIONS 0",
 			},
 		}
 		for i, test := range tests {

sql3/parser/ast_test.go

@@ -240,9 +240,7 @@ func TestCreateTableStatement_String(t *testing.T) {
 				Type: &parser.Type{Name: &parser.Ident{Name: "TIMESTAMP"}},
 				Constraints: []parser.Constraint{
 					&parser.TimeUnitConstraint{
-						Expr:      &parser.StringLit{Value: "s"},
-						Epoch:     pos(0),
-						EpochExpr: &parser.StringLit{Value: "2021-01-01T00:00:00Z"},
+						Expr: &parser.StringLit{Value: "s"},
 					},
 				},
 			},
@@ -255,7 +253,7 @@ func TestCreateTableStatement_String(t *testing.T) {
 		`intcol INTEGER MIN 100 MAX 1000, `+
 		`stringcol STRING CACHETYPE RANKED SIZE 10000, `+
 		`stringsetcol STRINGSET CACHETYPE RANKED SIZE 10000, `+
-		`timestampcol TIMESTAMP TIMEUNIT 's' EPOCH '2021-01-01T00:00:00Z'`+
+		`timestampcol TIMESTAMP TIMEUNIT 's'`+
 		`)`)
 }
 

sql3/parser/parser.go

@@ -860,15 +860,6 @@ func (p *Parser) parseTimeUnitConstraint(constraintPos Pos, name *Ident) (_ *Tim
 	} else {
 		return &cons, p.errorExpected(p.pos, p.tok, "literal")
 	}
-	if p.peek() == EPOCH {
-		cons.Epoch, _, _ = p.scan()
-
-		if isLiteralToken(p.peek()) {
-			cons.EpochExpr = p.mustParseLiteral()
-		} else {
-			return &cons, p.errorExpected(p.pos, p.tok, "literal")
-		}
-	}
 	return &cons, nil
 }
 

sql3/planner/expression.go

@@ -41,7 +41,6 @@ func coerceValue(sourceType parser.ExprDataType, targetType parser.ExprDataType,
 				return nil, sql3.NewErrInternalf("unexpected value type '%T'", value)
 			}
 			return pql.NewDecimal(val*int64(math.Pow(10, float64(t.Scale))), t.Scale), nil
-
 		case *parser.DataTypeTimestamp:
 			val, ok := value.(int64)
 			if !ok {
@@ -65,7 +64,6 @@ func coerceValue(sourceType parser.ExprDataType, targetType parser.ExprDataType,
 				return nil, sql3.NewErrInternalf("unexpected value type '%T'", value)
 			}
 			return pql.NewDecimal(int64(val)*int64(math.Pow(10, float64(t.Scale))), t.Scale), nil
-
 		case *parser.DataTypeTimestamp:
 			val, ok := value.(int64)
 			if !ok {
@@ -1574,6 +1572,8 @@ func (n *callPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er
 		return n.EvaluateFormat(currentRow)
 	case "CHARINDEX":
 		return n.EvaluateCharIndex(currentRow)
+	case "TOTIMESTAMP":
+		return n.EvaluateToTimestamp(currentRow)
 	case "STR":
 		return n.EvaluateStr(currentRow)
 	default:

sql3/planner/expressionanalyzercall.go

@@ -254,6 +254,8 @@ func (p *ExecutionPlanner) analyzeCallExpression(ctx context.Context, call *pars
 		return p.analyseFunctionFormat(call, scope)
 	case "CHARINDEX":
 		return p.analyseFunctionCharIndex(call, scope)
+	case "TOTIMESTAMP":
+		return p.analyzeFunctionToTimestamp(call, scope)
 	case "STR":
 		return p.analyseFunctionStr(call, scope)
 	default:

sql3/planner/expressiontypes.go

@@ -346,12 +346,12 @@ func typesAreAssignmentCompatible(targetType parser.ExprDataType, sourceType par
 		switch sourceType.(type) {
 		case *parser.DataTypeTimestamp:
 			return true
-		case *parser.DataTypeInt:
-			//could be a int convertable to a date
-			return true
 		case *parser.DataTypeString:
 			//could be a string parseable as a date
 			return true
+		case *parser.DataTypeInt:
+			//integers coerced to timestamp will be treated as time represented in number of seconds since unix epoch
+			return true
 		default:
 			return false
 		}

sql3/planner/inbuiltfunctionsdate.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 	"time"
 
+	featurebase "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/sql3"
 	"github.com/featurebasedb/featurebase/v3/sql3/parser"
 )
@@ -43,6 +44,33 @@ func (p *ExecutionPlanner) analyzeFunctionDatePart(call *parser.Call, scope pars
 	return call, nil
 }
 
+func (p *ExecutionPlanner) analyzeFunctionToTimestamp(call *parser.Call, scope parser.Statement) (parser.Expr, error) {
+	//param1 is the number to be converted to timestamp. This param is required.
+	//param2 is the time unit of the numeric value in param 1. This param is optional.
+	//ToTimestamp can be invoked with just param1.
+	if len(call.Args) != 1 && len(call.Args) != 2 {
+		return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 2, len(call.Args))
+	}
+
+	//param1 is a integer of type int64
+	param1Type := parser.NewDataTypeInt()
+	if !typesAreAssignmentCompatible(param1Type, call.Args[0].DataType()) {
+		return nil, sql3.NewErrParameterTypeMistmatch(call.Args[0].Pos().Line, call.Args[0].Pos().Column, call.Args[0].DataType().TypeDescription(), param1Type.TypeDescription())
+	}
+
+	//param2 is a string and it should be one of 's', 'ms', 'us', 'ns'.
+	//param2 is optional, will be defaulted to 's' if not supplied.
+	if len(call.Args) == 2 {
+		param2Type := parser.NewDataTypeString()
+		if !typesAreAssignmentCompatible(param2Type, call.Args[1].DataType()) {
+			return nil, sql3.NewErrParameterTypeMistmatch(call.Args[1].Pos().Line, call.Args[1].Pos().Column, call.Args[1].DataType().TypeDescription(), param2Type.TypeDescription())
+		}
+	}
+	//ToTimestamp returns a timestamp calculated from param1 using time unit passed in param 2
+	call.ResultDataType = parser.NewDataTypeTimestamp()
+	return call, nil
+}
+
 func (n *callPlanExpression) EvaluateDatepart(currentRow []interface{}) (interface{}, error) {
 	intervalEval, err := n.args[0].Evaluate(currentRow)
 	if err != nil {
@@ -121,3 +149,50 @@ func (n *callPlanExpression) EvaluateDatepart(currentRow []interface{}) (interfa
 	}
 
 }
+
+func (n *callPlanExpression) EvaluateToTimestamp(currentRow []interface{}) (interface{}, error) {
+	//retrieve param1, the number to be converted to timestamp
+	param1, err := n.args[0].Evaluate(currentRow)
+	if err != nil {
+		return nil, err
+	} else if param1 == nil {
+		//if the param1 is null silently return null timestamp value
+		return nil, nil
+	}
+	coercedParam1, err := coerceValue(n.args[0].Type(), parser.NewDataTypeInt(), param1, parser.Pos{Line: 0, Column: 0})
+	if err != nil {
+		//raise error if param 1 is not an integer. Should we return nil instead of raising error here? see note at return.
+		return nil, err
+	}
+	num, ok := coercedParam1.(int64)
+	if !ok {
+		//raise error if param 1 is not an integer. Should we return nil instead of raising error here? see note at return.
+		return nil, sql3.NewErrInternalf("unable to convert value")
+	}
+
+	//retrieve param2, time unit for param1, if not supplied default to seconds 's'.
+	var unit string = featurebase.TimeUnitSeconds
+	if len(n.args) == 2 {
+		param2, err := n.args[1].Evaluate(currentRow)
+		if err != nil {
+			//raise error if unable to retieve the argument for param2
+			return nil, err
+		}
+		coercedParam2, err := coerceValue(n.args[1].Type(), parser.NewDataTypeString(), param2, parser.Pos{Line: 0, Column: 0})
+		if err != nil {
+			//raise error if param2 is not a string
+			return nil, err
+		}
+		unit, ok = coercedParam2.(string)
+		if !ok {
+			//raise error if param2 is not a string
+			return nil, sql3.NewErrInternalf("unable to convert value")
+		}
+		if !featurebase.IsValidTimeUnit(unit) {
+			//raise error is param2 is not a valid time unit
+			return nil, sql3.NewErrCallParameterValueInvalid(0, 0, unit, "timeunit")
+		}
+	}
+	//should we throw error or return nil if the conversion fails? what is the desired behaviour when ToTimestamp errors for one bad record in a batch of thousands?
+	return featurebase.ValToTimestamp(unit, num)
+}

sql3/planner/opbulkinsert.go

@@ -320,7 +320,8 @@ func (i *bulkInsertSourceCSVRowIter) Next(ctx context.Context) (types.Row, error
 					return nil, sql3.NewErrTypeConversionOnMap(0, 0, evalValue, mapColumn.colType.TypeDescription())
 				}
 			} else {
-				result[idx] = time.UnixMilli(intVal).UTC()
+				//implicit conversion of int to timestamp will treat int as seconds since unix epoch
+				result[idx] = time.Unix(intVal, 0).UTC()
 			}
 
 		case *parser.DataTypeString:
@@ -651,7 +652,8 @@ func (i *bulkInsertSourceNDJsonRowIter) Next(ctx context.Context) (types.Row, er
 					case float64:
 						// if v is a whole number then make it an int
 						if v == float64(int64(v)) {
-							result[idx] = time.UnixMilli(int64(v)).UTC()
+							//implicit conversion of int to timestamp will treat int as seconds since unix epoch
+							result[idx] = time.Unix(int64(v), 0).UTC()
 						} else {
 							return nil, sql3.NewErrTypeConversionOnMap(0, 0, v, mapColumn.colType.TypeDescription())
 						}
@@ -1153,7 +1155,8 @@ func (i *bulkInsertSourceParquetRowIter) Next(ctx context.Context) (types.Row, e
 
 		case *parser.DataTypeTimestamp:
 			if intVal, ok := evalValue.(int64); ok {
-				result[idx] = time.UnixMilli(intVal).UTC()
+				//implicit conversion of int to timestamp will treat int as seconds since unix epoch
+				result[idx] = time.Unix(intVal, 0).UTC()
 			} else if stringVal, ok := evalValue.(string); ok {
 				if tm, err := time.ParseInLocation(time.RFC3339Nano, stringVal, time.UTC); err == nil {
 					result[idx] = tm

sql3/planner/opinsert.go

@@ -390,7 +390,26 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) {
 						return nil, errors.Wrapf(err, "converting timestamp to int64: %s", v)
 					}
 					row.Values[posVals[idx]] = i64
-
+				//integers passed as input for Timestamp fields will be treated as time represented in number of seconds since epoch defined for the field
+				//for timestamp fields created using SQL epoch will be defaulted to unix epoch
+				case int64:
+					// Convert the input seconds to target timeunit defined for the Timestamp field
+					// Add Base, which is the base epoch for Timestamp fields, to the input before saving
+					unit := fbbatch.TimeUnit(opts.TimeUnit)
+					i64 := opts.Base
+					switch unit {
+					case fbbatch.TimeUnitSeconds:
+						i64 = i64 + v
+					case fbbatch.TimeUnitMilliseconds:
+						i64 = i64 + (v * 1000)
+					case fbbatch.TimeUnitMicroseconds, fbbatch.TimeUnitUSeconds:
+						i64 = i64 + (v * 1000000)
+					case fbbatch.TimeUnitNanoseconds:
+						i64 = i64 + (v * 1000000000)
+					default:
+						return nil, errors.Wrapf(err, "unknown time unit: %s", unit)
+					}
+					row.Values[posVals[idx]] = i64
 				// nil is to support `null` values.
 				case nil:
 					row.Values[posVals[idx]] = eval

sql3/sql_complex_test.go

@@ -460,13 +460,13 @@ func TestPlanner_CoverCreateTable(t *testing.T) {
 			{
 				name:        "timestampcol",
 				typ:         "timestamp",
-				constraints: "timeunit 'ms' epoch '2021-01-01T00:00:00Z'",
+				constraints: "timeunit 'ms'",
 				expOptions: pilosa.FieldOptions{
-					Base:     1609459200000,
+					Base:     0,
 					Type:     "timestamp",
 					TimeUnit: "ms",
-					Min:      pql.NewDecimal(-63745055999000, 0),
-					Max:      pql.NewDecimal(251792841599000, 0),
+					Min:      pql.NewDecimal(-62135596799000, 0),
+					Max:      pql.NewDecimal(253402300799000, 0),
 				},
 			},
 			{
@@ -713,7 +713,7 @@ func TestPlanner_CreateTable(t *testing.T) {
 			_id id,
 			intcol int min 0 max 10000,
 			boolcol bool,
-			timestampcol timestamp timeunit 'ms' epoch '2010-01-01T00:00:00Z',
+			timestampcol timestamp timeunit 'ms',
 			decimalcol decimal(2),
 			stringcol string cachetype ranked size 1000,
 			stringsetcol stringset cachetype lru size 1000,
@@ -2957,7 +2957,7 @@ func TestPlanner_BulkInsertParquet(t *testing.T) {
 		now := time.Now()
 		simpleParquetMaker(t, tmpfile, 1, []tb{
 			{Name: "id", Type: arrow.PrimitiveTypes.Int64, Value: []int64{1}},
-			{Name: "unixtime", Type: arrow.PrimitiveTypes.Int64, Value: []int64{now.UnixMilli()}},
+			{Name: "unixtime", Type: arrow.PrimitiveTypes.Int64, Value: []int64{now.Unix()}},
 			{Name: "stringtime", Type: arrow.BinaryTypes.String, Value: []string{now.Format(time.RFC3339)}},
 		})
 

sql3/test/defs/defs_date_functions.go

@@ -1,5 +1,7 @@
 package defs
 
+import "time"
+
 // datepart tests
 var datePartTests = TableTest{
 
@@ -34,6 +36,30 @@ var datePartTests = TableTest{
 			),
 			ExpErr: "invalid value '1' for parameter 'interval'",
 		},
+		{
+			SQLs: sqls(
+				"select totimestamp()",
+			),
+			ExpErr: "count of formal parameters (2) does not match count of actual parameters (0)",
+		},
+		{
+			SQLs: sqls(
+				"select totimestamp('a')",
+			),
+			ExpErr: "an expression of type 'string' cannot be passed to a parameter of type 'int'",
+		},
+		{
+			SQLs: sqls(
+				"select totimestamp(1, 2)",
+			),
+			ExpErr: "an expression of type 'int' cannot be passed to a parameter of type 'string'",
+		},
+		{
+			SQLs: sqls(
+				"select totimestamp(1, 'x')",
+			),
+			ExpErr: "invalid value 'x' for parameter 'timeunit'",
+		},
 		{
 			SQLs: sqls(
 				"select _id, datepart('yy', ts) from dateparttests",
@@ -177,5 +203,43 @@ var datePartTests = TableTest{
 			),
 			Compare: CompareExactUnordered,
 		},
+		{
+			//test datepart(timestamp, part) for implicit conversion of integer value passed as argument to timestamp param
+			SQLs: sqls(
+				"select datepart('yy', 0) as \"yy\", datepart('m', 0) as \"m\", datepart('d', 0) as \"d\"",
+			),
+			ExpHdrs: hdrs(
+				hdr("yy", fldTypeInt),
+				hdr("m", fldTypeInt),
+				hdr("d", fldTypeInt),
+			),
+			ExpRows: rows(
+				row(int64(1970), int64(1), int64(1)),
+			),
+			Compare: CompareExactUnordered,
+		},
+		{
+			//test ToTimestamp(num, timeunit) for all possible time unit values
+			SQLs: sqls(
+				"select totimestamp(1000) as \"default\", totimestamp(1000, 's') as \"s\", totimestamp(1000000, 'ms') as \"ms\", totimestamp(1000000000, 'us') as \"us\", totimestamp(1000000000, 'µs') as \"µs\", totimestamp(1000000000000, 'ns') as \"ns\"",
+			),
+			ExpHdrs: hdrs(
+				hdr("default", fldTypeTimestamp),
+				hdr("s", fldTypeTimestamp),
+				hdr("ms", fldTypeTimestamp),
+				hdr("us", fldTypeTimestamp),
+				hdr("µs", fldTypeTimestamp),
+				hdr("ns", fldTypeTimestamp),
+			),
+			ExpRows: rows(
+				row(time.Unix(1000, 0).UTC(),
+					time.Unix(1000, 0).UTC(),
+					time.UnixMilli(1000000).UTC(),
+					time.UnixMicro(1000000000).UTC(),
+					time.UnixMicro(1000000000).UTC(),
+					time.Unix(0, 1000000000000).UTC()),
+			),
+			Compare: CompareExactUnordered,
+		},
 	},
 }

sql3/test/defs/defs_inserts.go

@@ -159,7 +159,7 @@ var insertTimestampTest = TableTest{
 	SQLTests: []SQLTest{
 		{
 			SQLs: sqls(
-				"CREATE TABLE insertTimestampTest (_id id, time timestamp timeunit 'ms' epoch '2022-01-01T00:00:00Z', ids idset, strings stringset);",
+				"CREATE TABLE insertTimestampTest (_id id, time timestamp timeunit 'ms', ids idset, strings stringset);",
 			),
 			ExpHdrs: hdrs(),
 			ExpRows: rows(),
@@ -173,6 +173,14 @@ var insertTimestampTest = TableTest{
 			ExpRows: rows(),
 			Compare: CompareExactUnordered,
 		},
+		{
+			SQLs: sqls(
+				"INSERT INTO insertTimestampTest(_id, time, ids, strings) VALUES (2, 1672531200, [6 , 1, 9], ['red', 'blue', 'green']);",
+			),
+			ExpHdrs: hdrs(),
+			ExpRows: rows(),
+			Compare: CompareExactUnordered,
+		},
 		{
 			SQLs: sqls(
 				"select time from insertTimestampTest;",
@@ -182,6 +190,7 @@ var insertTimestampTest = TableTest{
 			),
 			ExpRows: rows(
 				row(timestampFromString("2023-01-01T00:00:00Z")),
+				row(timestampFromString("2023-01-01T00:00:00Z")),
 			),
 			Compare:        CompareExactUnordered,
 			SortStringKeys: true,

sql3/test/defs/defs_timestamp_literals.go

@@ -18,7 +18,7 @@ var timestampLiterals = TableTest{
 		{
 			// InsertWithCurrentTimestamp
 			SQLs: sqls(
-				"insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (4, 40, 400, 10.12, current_timestamp, ['A', 'B', 'C'], [1, 2, 3])",
+				"insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (1, 40, 400, 10.12, current_timestamp, ['A', 'B', 'C'], [1, 2, 3])",
 			),
 			ExpHdrs: hdrs(),
 			ExpRows: rows(),
@@ -27,11 +27,44 @@ var timestampLiterals = TableTest{
 		{
 			// InsertWithCurrentDate
 			SQLs: sqls(
-				"insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (4, 40, 400, 10.12, current_date, ['A', 'B', 'C'], [1, 2, 3])",
+				"insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (2, 40, 400, 10.12, current_date, ['A', 'B', 'C'], [1, 2, 3])",
 			),
 			ExpHdrs: hdrs(),
 			ExpRows: rows(),
 			Compare: CompareExactUnordered,
 		},
+		{
+			// Insert literal 0 into a timestamp, it should be stored as 1970-01-01 00:00:00 +0000 UTC (unix epoch base value)
+			SQLs: sqls(
+				"insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (3, 40, 400, 10.12, 0, ['A', 'B', 'C'], [1, 2, 3])",
+			),
+			ExpHdrs: hdrs(),
+			ExpRows: rows(),
+			Compare: CompareExactUnordered,
+		},
+		{
+			// Insert literal -86400 into a timestamp, it should be stored as 1969-12-31 00:00:00 +0000 UTC (unix epoch base value)
+			SQLs: sqls(
+				"insert into testtimestampliterals (_id, a, b, d, ts, event, ievent) values (4, 40, 400, 10.12, -86400, ['A', 'B', 'C'], [1, 2, 3])",
+			),
+			ExpHdrs: hdrs(),
+			ExpRows: rows(),
+			Compare: CompareExactUnordered,
+		},
+		{
+			//compare test is done only for integer test cases (_id in (3,4), because only for these cases we have a determinate year(1970, 1969) to look for.
+			SQLs: sqls(
+				"select _id, datepart('yy', ts) as \"yy\" from testtimestampliterals where _id in (3,4)",
+			),
+			ExpHdrs: hdrs(
+				hdr("_id", fldTypeID),
+				hdr("yy", fldTypeInt),
+			),
+			ExpRows: rows(
+				row(int64(3), int64(1970)),
+				row(int64(4), int64(1969)),
+			),
+			Compare: CompareExactUnordered,
+		},
 	},
 }