From 0927240956cf1f2fbc3fe225a277a837a5c1eba0 Mon Sep 17 00:00:00 2001 From: nagamocha3000 Date: Sat, 6 Feb 2021 01:54:45 +0300 Subject: [PATCH] Add parsing for partial time inputs --- executor_test.go | 129 ++++++++++++++++++++++++++++++++++++++++++ time.go | 123 +++++++++++++++++++++++++++++++++++++++- time_internal_test.go | 60 ++++++++++++++++++++ 3 files changed, 311 insertions(+), 1 deletion(-) diff --git a/executor_test.go b/executor_test.go index 94a4ceade..3cb51cac7 100644 --- a/executor_test.go +++ b/executor_test.go @@ -27,6 +27,7 @@ import ( "math/rand" "os" "reflect" + "sort" "strconv" "strings" "testing" @@ -6840,6 +6841,134 @@ func TestVariousQueries(t *testing.T) { t.Parallel() variousQueries(t, clusterSize) + variousQueriesOnTimeFields(t, clusterSize) + }) + } +} + +// tests for abbreviating time values in queries +func variousQueriesOnTimeFields(t *testing.T, clusterSize int) { + c := test.MustRunCluster(t, clusterSize) + defer c.Close() + + ts := func(t time.Time) int64 { + return t.Unix() * 1e+9 + } + + // generic index + // worth noting, since we are using YMDH resolution, both C4 & C5 + // get binned to the same hour + c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f1", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + c.ImportTimeQuantumKey(t, "t_index", "f1", []test.TimeQuantumKey{ + // from edge cases + {ColKey: "C1", RowKey: "R1", Ts: ts(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))}, + {ColKey: "C2", RowKey: "R2", Ts: ts(time.Date(2019, 8, 1, 0, 0, 0, 0, time.UTC))}, + {ColKey: "C3", RowKey: "R3", Ts: ts(time.Date(2019, 8, 4, 0, 0, 0, 0, time.UTC))}, + {ColKey: "C4", RowKey: "R4", Ts: ts(time.Date(2019, 8, 4, 14, 0, 0, 0, time.UTC))}, + {ColKey: "C5", RowKey: "R5", Ts: ts(time.Date(2019, 8, 4, 14, 36, 0, 0, time.UTC))}, + // to edge cases + {ColKey: "C6", RowKey: "R6", Ts: ts(time.Date(2019, 8, 4, 16, 0, 0, 0, time.UTC))}, + {ColKey: "C7", RowKey: "R7", Ts: ts(time.Date(2019, 8, 5, 0, 0, 0, 0, time.UTC))}, + {ColKey: "C8", RowKey: "R8", Ts: ts(time.Date(2019, 12, 1, 0, 0, 0, 0, time.UTC))}, + {ColKey: "C9", RowKey: "R9", Ts: ts(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))}, + }) + + // in this field, all columns have the same row value to simplify test queries for Row + c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f2", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + c.ImportTimeQuantumKey(t, "t_index", "f2", []test.TimeQuantumKey{ + // from + {ColKey: "C1", RowKey: "R", Ts: ts(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))}, + {ColKey: "C2", RowKey: "R", Ts: ts(time.Date(2019, 8, 1, 0, 0, 0, 0, time.UTC))}, + {ColKey: "C3", RowKey: "R", Ts: ts(time.Date(2019, 8, 4, 0, 0, 0, 0, time.UTC))}, + {ColKey: "C4", RowKey: "R", Ts: ts(time.Date(2019, 8, 4, 14, 0, 0, 0, time.UTC))}, + {ColKey: "C5", RowKey: "R", Ts: ts(time.Date(2019, 8, 4, 14, 36, 0, 0, time.UTC))}, + // to + {ColKey: "C6", RowKey: "R", Ts: ts(time.Date(2019, 8, 4, 16, 0, 0, 0, time.UTC))}, + {ColKey: "C7", RowKey: "R", Ts: ts(time.Date(2019, 8, 5, 0, 0, 0, 0, time.UTC))}, + {ColKey: "C8", RowKey: "R", Ts: ts(time.Date(2019, 12, 1, 0, 0, 0, 0, time.UTC))}, + {ColKey: "C9", RowKey: "R", Ts: ts(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))}, + }) + + splitSortBackToCSV := func(csvStr string) string { + ss := strings.Split(csvStr[:len(csvStr)-1], "\n") + sort.Strings(ss) + return strings.Join(ss, "\n") + "\n" + } + + toCSV := func(s string) string { + return strings.Join(strings.Split(s, " "), "\n") + "\n" + } + + type testCase struct { + query string + qrVerifier func(t *testing.T, resp pilosa.QueryResponse) + csvVerifier string + } + + tests := []testCase{ + // Rows + { + query: `Rows(f1, from='2019-08-04T14:36', to='2019-08-04T16:00')`, + csvVerifier: toCSV("R4 R5"), + }, + { + query: `Rows(f1, from='2019-08-04T14', to='2019-08-04T17:00')`, + csvVerifier: toCSV("R4 R5 R6"), + }, + { + query: `Rows(f1, from='2019-08-04', to='2019-08-05')`, + csvVerifier: toCSV("R3 R4 R5 R6"), + }, + { + query: `Rows(f1, from='2019-08', to='2019-12')`, + csvVerifier: toCSV("R2 R3 R4 R5 R6 R7"), + }, + { + query: `Rows(f1, from='2019', to='2020')`, + csvVerifier: toCSV("R1 R2 R3 R4 R5 R6 R7 R8"), + }, + // Row + { + query: `Row(f2='R', from='2019-08-04T14:36', to='2019-08-04T16:00')`, + csvVerifier: toCSV("C4 C5"), + }, + { + query: `Row(f2='R', from='2019-08-04T14', to='2019-08-04T17:00')`, + csvVerifier: toCSV("C4 C5 C6"), + }, + { + query: `Row(f2='R', from='2019-08-04', to='2019-08-05')`, + csvVerifier: toCSV("C3 C4 C5 C6"), + }, + { + query: `Row(f2='R', from='2019-08', to='2019-12')`, + csvVerifier: toCSV("C2 C3 C4 C5 C6 C7"), + }, + { + query: `Row(f2='R', from='2019', to='2020')`, + csvVerifier: toCSV("C1 C2 C3 C4 C5 C6 C7 C8"), + }, + } + + for i, tst := range tests { + t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) { + resp := c.Query(t, "t_index", tst.query) + tr := c.QueryGRPC(t, "t_index", tst.query) + if tst.qrVerifier != nil { + tst.qrVerifier(t, resp) + } + csvString, err := tableResponseToCSVString(tr) + if err != nil { + t.Fatal(err) + } + // verify everything after header + got := splitSortBackToCSV(csvString[strings.Index(csvString, "\n")+1:]) + if got != tst.csvVerifier { + t.Errorf("expected:\n%s\ngot:\n%s", tst.csvVerifier, got) + } + + // TODO: add HTTP and Postgres and ability to convert + // those results to CSV to run through CSV verifier }) } } diff --git a/time.go b/time.go index 75ca543ea..b87774f1a 100644 --- a/time.go +++ b/time.go @@ -17,7 +17,9 @@ package pilosa import ( "errors" "fmt" + "regexp" "sort" + "strconv" "strings" "time" ) @@ -223,7 +225,10 @@ func parseTime(t interface{}) (time.Time, error) { switch v := t.(type) { case string: if calcTime, err = time.Parse(TimeFormat, v); err != nil { - return time.Time{}, errors.New("cannot parse string time") + // if the default parsing fails, check if user tried to + // supply partial time eg year and month + calcTime, err := parsePartialTime(v) + return calcTime, err } case int64: calcTime = time.Unix(v, 0).UTC() @@ -233,6 +238,122 @@ func parseTime(t interface{}) (time.Time, error) { return calcTime, nil } +// parsePartialTime parses strings where the time provided is only partial +// eg given 2006-02, it extracts the year and month and the rest of the +// components are set to the default values. The time must have the format +// used in parseTime. The year must be present. The rest of the components are +// optional but if a component is present in the input, this implies that all +// the preceding components are also specified. For example, if the hour is provided +// then the day, month and year must be present. This function could and should be +// simplified +func parsePartialTime(t string) (time.Time, error) { + // helper parseCustomHourMinute parses strings of the form HH:MM to + // hour and minute component + parseHourMinute := func(t string) (hour, minute int, err error) { + // time should have the format HH:MM + subStrings := strings.Split(t, ":") + switch len(subStrings) { + case 2: + // has minutes + minute, err = strconv.Atoi(subStrings[1]) + if err != nil { + return -1, -1, errors.New("Invalid Time") + } + fallthrough + case 1: + hour, err = strconv.Atoi(subStrings[0]) + if err != nil { + return -1, -1, errors.New("Invalid Time") + } + default: + return -1, -1, errors.New("Invalid Time") + } + + return + } + // helper trim function + trim := func(subMatches []string) (filtered []string, err error) { + restAreEmpty := func(ss []string) bool { + for _, s := range ss { + if s != "" { + return false + } + } + return true + } + if len(subMatches) <= 1 { + return nil, errors.New("Invalid time") + } + // ignore full match which is at index 0 + subMatches = subMatches[1:] // ignore full match which is at index 0 + for i, s := range subMatches { + if s != "" { + if i > 0 { + s = s[1:] // remove preceding hyphen or T + } + filtered = append(filtered, s) + } else { + // rest must be empty for date-time to be valid + if !restAreEmpty(subMatches[i:]) { + return nil, errors.New("Invalid date-time") + } + break + } + } + return filtered, nil + } + + var errInvalidTime error = errors.New("cannot parse string time") + var regex = regexp.MustCompile(`^(\d{4})(-\d{2})?(-\d{2})?(T.+)?$`) + subMatches := regex.FindStringSubmatch(t) + subMatches, err := trim(subMatches) + if err != nil { + return time.Time{}, errInvalidTime + } + // defaults + var ( + yr int + month = time.January + day = 1 + hour = 0 + min = 0 + ) + // year must be set, the rest are optional + switch len(subMatches) { + case 4: + // time + hour, min, err = parseHourMinute(subMatches[3]) + if err != nil { + return time.Time{}, errInvalidTime + } + fallthrough + case 3: + // day + day, err = strconv.Atoi(subMatches[2]) + if err != nil { + return time.Time{}, errInvalidTime + } + fallthrough + case 2: + // month + monthNum, err := strconv.Atoi(subMatches[1]) + month = time.Month(monthNum) + if err != nil { + return time.Time{}, errInvalidTime + } + fallthrough + case 1: + // year + yr, err = strconv.Atoi(subMatches[0]) + if err != nil { + return time.Time{}, errInvalidTime + } + default: + return time.Time{}, errInvalidTime + } + return time.Date(yr, month, day, hour, min, 0, 0, time.UTC), nil +} + // minMaxViews returns the min and max view from a list of views // with a time quantum taken into consideration. It assumes that // all views represent the same base view name (the logic depends diff --git a/time_internal_test.go b/time_internal_test.go index 58079380b..8356d5dbe 100644 --- a/time_internal_test.go +++ b/time_internal_test.go @@ -319,3 +319,63 @@ func parseTimeQuantum(v string) (TimeQuantum, error) { } return q, nil } + +func TestParsePartialTime(t *testing.T) { + // test handling of valud inputs + testCases := []struct { + userInput string + expectedTime time.Time + }{ + { + "2006", + time.Date(2006, 1, 1, 0, 0, 0, 0, time.UTC), + }, + { + "2006-07", + time.Date(2006, time.July, 1, 0, 0, 0, 0, time.UTC), + }, + { + "2006-07-02", + time.Date(2006, time.July, 2, 0, 0, 0, 0, time.UTC), + }, + { + "2006-07-02T15", + time.Date(2006, time.July, 2, 15, 0, 0, 0, time.UTC), + }, + { + "2006-07-02T15:04", + time.Date(2006, time.July, 2, 15, 4, 0, 0, time.UTC), + }, + } + for _, tc := range testCases { + got, err := parsePartialTime(tc.userInput) + if err != nil { + t.Errorf("expected nil error given parsing for '%s'", tc.userInput) + } + if got != tc.expectedTime { + t.Errorf("expected %v, got %v", tc.expectedTime, got) + } + } + + // test handling of invalid inputs + invalidInputs := []string{ + " 2006-01-02 ", + " foo-bar ", + "2006-", + "2006-01-", + "2006-01-02T", + "2006T", + "2006T04", + "01-02", + "2006-01T04", + "2006-01T04:", + "2006-01T:04", + } + for _, invalidInput := range invalidInputs { + _, err := parsePartialTime(invalidInput) + if err == nil { + t.Errorf("for input '%s', error on parse expected", invalidInput) + } + } + +}