mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-05 16:15:56 +00:00
Merge pull request #1395 from nagamocha3000/fix-core-78
CORE-78 Add parsing for partial time inputs
This commit is contained in:
nagamocha3000
GitHub
commit
4dfd713189
3 changed files with 311 additions and 1 deletions
129
executor_test.go
129
executor_test.go
|
|
@ -27,6 +27,7 @@ import (
|
|||
"math/rand"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
|
@ -6840,6 +6841,134 @@ func TestVariousQueries(t *testing.T) {
|
|||
t.Parallel()
|
||||
|
||||
variousQueries(t, clusterSize)
|
||||
variousQueriesOnTimeFields(t, clusterSize)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// tests for abbreviating time values in queries
|
||||
func variousQueriesOnTimeFields(t *testing.T, clusterSize int) {
|
||||
c := test.MustRunCluster(t, clusterSize)
|
||||
defer c.Close()
|
||||
|
||||
ts := func(t time.Time) int64 {
|
||||
return t.Unix() * 1e+9
|
||||
}
|
||||
|
||||
// generic index
|
||||
// worth noting, since we are using YMDH resolution, both C4 & C5
|
||||
// get binned to the same hour
|
||||
c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f1", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
|
||||
c.ImportTimeQuantumKey(t, "t_index", "f1", []test.TimeQuantumKey{
|
||||
// from edge cases
|
||||
{ColKey: "C1", RowKey: "R1", Ts: ts(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C2", RowKey: "R2", Ts: ts(time.Date(2019, 8, 1, 0, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C3", RowKey: "R3", Ts: ts(time.Date(2019, 8, 4, 0, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C4", RowKey: "R4", Ts: ts(time.Date(2019, 8, 4, 14, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C5", RowKey: "R5", Ts: ts(time.Date(2019, 8, 4, 14, 36, 0, 0, time.UTC))},
|
||||
// to edge cases
|
||||
{ColKey: "C6", RowKey: "R6", Ts: ts(time.Date(2019, 8, 4, 16, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C7", RowKey: "R7", Ts: ts(time.Date(2019, 8, 5, 0, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C8", RowKey: "R8", Ts: ts(time.Date(2019, 12, 1, 0, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C9", RowKey: "R9", Ts: ts(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))},
|
||||
})
|
||||
|
||||
// in this field, all columns have the same row value to simplify test queries for Row
|
||||
c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f2", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
|
||||
c.ImportTimeQuantumKey(t, "t_index", "f2", []test.TimeQuantumKey{
|
||||
// from
|
||||
{ColKey: "C1", RowKey: "R", Ts: ts(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C2", RowKey: "R", Ts: ts(time.Date(2019, 8, 1, 0, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C3", RowKey: "R", Ts: ts(time.Date(2019, 8, 4, 0, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C4", RowKey: "R", Ts: ts(time.Date(2019, 8, 4, 14, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C5", RowKey: "R", Ts: ts(time.Date(2019, 8, 4, 14, 36, 0, 0, time.UTC))},
|
||||
// to
|
||||
{ColKey: "C6", RowKey: "R", Ts: ts(time.Date(2019, 8, 4, 16, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C7", RowKey: "R", Ts: ts(time.Date(2019, 8, 5, 0, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C8", RowKey: "R", Ts: ts(time.Date(2019, 12, 1, 0, 0, 0, 0, time.UTC))},
|
||||
{ColKey: "C9", RowKey: "R", Ts: ts(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))},
|
||||
})
|
||||
|
||||
splitSortBackToCSV := func(csvStr string) string {
|
||||
ss := strings.Split(csvStr[:len(csvStr)-1], "\n")
|
||||
sort.Strings(ss)
|
||||
return strings.Join(ss, "\n") + "\n"
|
||||
}
|
||||
|
||||
toCSV := func(s string) string {
|
||||
return strings.Join(strings.Split(s, " "), "\n") + "\n"
|
||||
}
|
||||
|
||||
type testCase struct {
|
||||
query string
|
||||
qrVerifier func(t *testing.T, resp pilosa.QueryResponse)
|
||||
csvVerifier string
|
||||
}
|
||||
|
||||
tests := []testCase{
|
||||
// Rows
|
||||
{
|
||||
query: `Rows(f1, from='2019-08-04T14:36', to='2019-08-04T16:00')`,
|
||||
csvVerifier: toCSV("R4 R5"),
|
||||
},
|
||||
{
|
||||
query: `Rows(f1, from='2019-08-04T14', to='2019-08-04T17:00')`,
|
||||
csvVerifier: toCSV("R4 R5 R6"),
|
||||
},
|
||||
{
|
||||
query: `Rows(f1, from='2019-08-04', to='2019-08-05')`,
|
||||
csvVerifier: toCSV("R3 R4 R5 R6"),
|
||||
},
|
||||
{
|
||||
query: `Rows(f1, from='2019-08', to='2019-12')`,
|
||||
csvVerifier: toCSV("R2 R3 R4 R5 R6 R7"),
|
||||
},
|
||||
{
|
||||
query: `Rows(f1, from='2019', to='2020')`,
|
||||
csvVerifier: toCSV("R1 R2 R3 R4 R5 R6 R7 R8"),
|
||||
},
|
||||
// Row
|
||||
{
|
||||
query: `Row(f2='R', from='2019-08-04T14:36', to='2019-08-04T16:00')`,
|
||||
csvVerifier: toCSV("C4 C5"),
|
||||
},
|
||||
{
|
||||
query: `Row(f2='R', from='2019-08-04T14', to='2019-08-04T17:00')`,
|
||||
csvVerifier: toCSV("C4 C5 C6"),
|
||||
},
|
||||
{
|
||||
query: `Row(f2='R', from='2019-08-04', to='2019-08-05')`,
|
||||
csvVerifier: toCSV("C3 C4 C5 C6"),
|
||||
},
|
||||
{
|
||||
query: `Row(f2='R', from='2019-08', to='2019-12')`,
|
||||
csvVerifier: toCSV("C2 C3 C4 C5 C6 C7"),
|
||||
},
|
||||
{
|
||||
query: `Row(f2='R', from='2019', to='2020')`,
|
||||
csvVerifier: toCSV("C1 C2 C3 C4 C5 C6 C7 C8"),
|
||||
},
|
||||
}
|
||||
|
||||
for i, tst := range tests {
|
||||
t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) {
|
||||
resp := c.Query(t, "t_index", tst.query)
|
||||
tr := c.QueryGRPC(t, "t_index", tst.query)
|
||||
if tst.qrVerifier != nil {
|
||||
tst.qrVerifier(t, resp)
|
||||
}
|
||||
csvString, err := tableResponseToCSVString(tr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// verify everything after header
|
||||
got := splitSortBackToCSV(csvString[strings.Index(csvString, "\n")+1:])
|
||||
if got != tst.csvVerifier {
|
||||
t.Errorf("expected:\n%s\ngot:\n%s", tst.csvVerifier, got)
|
||||
}
|
||||
|
||||
// TODO: add HTTP and Postgres and ability to convert
|
||||
// those results to CSV to run through CSV verifier
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
123
time.go
123
time.go
|
|
@ -17,7 +17,9 @@ package pilosa
|
|||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
|
@ -223,7 +225,10 @@ func parseTime(t interface{}) (time.Time, error) {
|
|||
switch v := t.(type) {
|
||||
case string:
|
||||
if calcTime, err = time.Parse(TimeFormat, v); err != nil {
|
||||
return time.Time{}, errors.New("cannot parse string time")
|
||||
// if the default parsing fails, check if user tried to
|
||||
// supply partial time eg year and month
|
||||
calcTime, err := parsePartialTime(v)
|
||||
return calcTime, err
|
||||
}
|
||||
case int64:
|
||||
calcTime = time.Unix(v, 0).UTC()
|
||||
|
|
@ -233,6 +238,122 @@ func parseTime(t interface{}) (time.Time, error) {
|
|||
return calcTime, nil
|
||||
}
|
||||
|
||||
// parsePartialTime parses strings where the time provided is only partial
|
||||
// eg given 2006-02, it extracts the year and month and the rest of the
|
||||
// components are set to the default values. The time must have the format
|
||||
// used in parseTime. The year must be present. The rest of the components are
|
||||
// optional but if a component is present in the input, this implies that all
|
||||
// the preceding components are also specified. For example, if the hour is provided
|
||||
// then the day, month and year must be present. This function could and should be
|
||||
// simplified
|
||||
func parsePartialTime(t string) (time.Time, error) {
|
||||
// helper parseCustomHourMinute parses strings of the form HH:MM to
|
||||
// hour and minute component
|
||||
parseHourMinute := func(t string) (hour, minute int, err error) {
|
||||
// time should have the format HH:MM
|
||||
subStrings := strings.Split(t, ":")
|
||||
switch len(subStrings) {
|
||||
case 2:
|
||||
// has minutes
|
||||
minute, err = strconv.Atoi(subStrings[1])
|
||||
if err != nil {
|
||||
return -1, -1, errors.New("Invalid Time")
|
||||
}
|
||||
fallthrough
|
||||
case 1:
|
||||
hour, err = strconv.Atoi(subStrings[0])
|
||||
if err != nil {
|
||||
return -1, -1, errors.New("Invalid Time")
|
||||
}
|
||||
default:
|
||||
return -1, -1, errors.New("Invalid Time")
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
// helper trim function
|
||||
trim := func(subMatches []string) (filtered []string, err error) {
|
||||
restAreEmpty := func(ss []string) bool {
|
||||
for _, s := range ss {
|
||||
if s != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
if len(subMatches) <= 1 {
|
||||
return nil, errors.New("Invalid time")
|
||||
}
|
||||
// ignore full match which is at index 0
|
||||
subMatches = subMatches[1:] // ignore full match which is at index 0
|
||||
for i, s := range subMatches {
|
||||
if s != "" {
|
||||
if i > 0 {
|
||||
s = s[1:] // remove preceding hyphen or T
|
||||
}
|
||||
filtered = append(filtered, s)
|
||||
} else {
|
||||
// rest must be empty for date-time to be valid
|
||||
if !restAreEmpty(subMatches[i:]) {
|
||||
return nil, errors.New("Invalid date-time")
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return filtered, nil
|
||||
}
|
||||
|
||||
var errInvalidTime error = errors.New("cannot parse string time")
|
||||
var regex = regexp.MustCompile(`^(\d{4})(-\d{2})?(-\d{2})?(T.+)?$`)
|
||||
subMatches := regex.FindStringSubmatch(t)
|
||||
subMatches, err := trim(subMatches)
|
||||
if err != nil {
|
||||
return time.Time{}, errInvalidTime
|
||||
}
|
||||
// defaults
|
||||
var (
|
||||
yr int
|
||||
month = time.January
|
||||
day = 1
|
||||
hour = 0
|
||||
min = 0
|
||||
)
|
||||
// year must be set, the rest are optional
|
||||
switch len(subMatches) {
|
||||
case 4:
|
||||
// time
|
||||
hour, min, err = parseHourMinute(subMatches[3])
|
||||
if err != nil {
|
||||
return time.Time{}, errInvalidTime
|
||||
}
|
||||
fallthrough
|
||||
case 3:
|
||||
// day
|
||||
day, err = strconv.Atoi(subMatches[2])
|
||||
if err != nil {
|
||||
return time.Time{}, errInvalidTime
|
||||
}
|
||||
fallthrough
|
||||
case 2:
|
||||
// month
|
||||
monthNum, err := strconv.Atoi(subMatches[1])
|
||||
month = time.Month(monthNum)
|
||||
if err != nil {
|
||||
return time.Time{}, errInvalidTime
|
||||
}
|
||||
fallthrough
|
||||
case 1:
|
||||
// year
|
||||
yr, err = strconv.Atoi(subMatches[0])
|
||||
if err != nil {
|
||||
return time.Time{}, errInvalidTime
|
||||
}
|
||||
default:
|
||||
return time.Time{}, errInvalidTime
|
||||
}
|
||||
return time.Date(yr, month, day, hour, min, 0, 0, time.UTC), nil
|
||||
}
|
||||
|
||||
// minMaxViews returns the min and max view from a list of views
|
||||
// with a time quantum taken into consideration. It assumes that
|
||||
// all views represent the same base view name (the logic depends
|
||||
|
|
|
|||
|
|
@ -319,3 +319,63 @@ func parseTimeQuantum(v string) (TimeQuantum, error) {
|
|||
}
|
||||
return q, nil
|
||||
}
|
||||
|
||||
func TestParsePartialTime(t *testing.T) {
|
||||
// test handling of valud inputs
|
||||
testCases := []struct {
|
||||
userInput string
|
||||
expectedTime time.Time
|
||||
}{
|
||||
{
|
||||
"2006",
|
||||
time.Date(2006, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
"2006-07",
|
||||
time.Date(2006, time.July, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
"2006-07-02",
|
||||
time.Date(2006, time.July, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
"2006-07-02T15",
|
||||
time.Date(2006, time.July, 2, 15, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
"2006-07-02T15:04",
|
||||
time.Date(2006, time.July, 2, 15, 4, 0, 0, time.UTC),
|
||||
},
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
got, err := parsePartialTime(tc.userInput)
|
||||
if err != nil {
|
||||
t.Errorf("expected nil error given parsing for '%s'", tc.userInput)
|
||||
}
|
||||
if got != tc.expectedTime {
|
||||
t.Errorf("expected %v, got %v", tc.expectedTime, got)
|
||||
}
|
||||
}
|
||||
|
||||
// test handling of invalid inputs
|
||||
invalidInputs := []string{
|
||||
" 2006-01-02 ",
|
||||
" foo-bar ",
|
||||
"2006-",
|
||||
"2006-01-",
|
||||
"2006-01-02T",
|
||||
"2006T",
|
||||
"2006T04",
|
||||
"01-02",
|
||||
"2006-01T04",
|
||||
"2006-01T04:",
|
||||
"2006-01T:04",
|
||||
}
|
||||
for _, invalidInput := range invalidInputs {
|
||||
_, err := parsePartialTime(invalidInput)
|
||||
if err == nil {
|
||||
t.Errorf("for input '%s', error on parse expected", invalidInput)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue