featurebase/time.go

// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

import (
	"errors"
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

// ErrInvalidTimeQuantum is returned when parsing a time quantum.
var ErrInvalidTimeQuantum = errors.New("invalid time quantum")

// TimeQuantum represents a time granularity for time-based bitmaps.
type TimeQuantum string

// HasYear returns true if the quantum contains a 'Y' unit.
func (q TimeQuantum) HasYear() bool { return strings.ContainsRune(string(q), 'Y') }

// HasMonth returns true if the quantum contains a 'M' unit.
func (q TimeQuantum) HasMonth() bool { return strings.ContainsRune(string(q), 'M') }

// HasDay returns true if the quantum contains a 'D' unit.
func (q TimeQuantum) HasDay() bool { return strings.ContainsRune(string(q), 'D') }

// HasHour returns true if the quantum contains a 'H' unit.
func (q TimeQuantum) HasHour() bool { return strings.ContainsRune(string(q), 'H') }

func (q TimeQuantum) Granularity() rune {
	var g rune
	for _, g = range q {
	}
	return g
}

// Valid returns true if q is a valid time quantum value.
func (q TimeQuantum) Valid() bool {
	switch q {
	case "Y", "YM", "YMD", "YMDH",
		"M", "MD", "MDH",
		"D", "DH",
		"H",
		"":
		return true
	default:
		return false
	}
}

// The following methods are required to implement pflag Value interface.

// Set sets the time quantum value.
func (q *TimeQuantum) Set(value string) error {
	*q = TimeQuantum(value)
	return nil
}

func (q TimeQuantum) String() string {
	return string(q)
}

// Type returns the type of a time quantum value.
func (q TimeQuantum) Type() string {
	return "TimeQuantum"
}

// viewByTimeUnit returns the view name for time with a given quantum unit.
func viewByTimeUnit(name string, t time.Time, unit rune) string {
	switch unit {
	case 'Y':
		return fmt.Sprintf("%s_%s", name, t.Format("2006"))
	case 'M':
		return fmt.Sprintf("%s_%s", name, t.Format("200601"))
	case 'D':
		return fmt.Sprintf("%s_%s", name, t.Format("20060102"))
	case 'H':
		return fmt.Sprintf("%s_%s", name, t.Format("2006010215"))
	default:
		return ""
	}
}

// YYYYMMDDHH lengths. Note that this is a []int, not a map[byte]int, so
// the lookups can be cheaper.
var lengthsByQuantum = []int{
	'Y': 4,
	'M': 6,
	'D': 8,
	'H': 10,
}

// viewsByTimeInto computes the list of views for a given time. It expects
// to be given an initial buffer of the form `name_YYYYMMDDHH`, and a slice
// of []bytes. This allows us to reuse the buffer for all the sub-buffers,
// and also to reuse the slice of slices, to eliminate all those allocations.
// This might seem crazy, but even including the JSON parsing and all the
// disk activity, the straightforward viewsByTime implementation was 25%
// of runtime in an ingest test.
func viewsByTimeInto(fullBuf []byte, into [][]byte, t time.Time, q TimeQuantum) [][]byte {
	l := len(fullBuf) - 10
	date := fullBuf[l : l+10]
	y, m, d := t.Date()
	h := t.Hour()
	// Did you know that Sprintf, Printf, and other things like that all
	// do allocations, and that doing allocations in a tight loop like this
	// is stunningly expensive? viewsByTime was 25% of an ingest test's
	// total CPU, not counting the garbage collector overhead. This is about
	// 3%. No, I'm not totally sure that justifies it.
	if y < 1000 {
		ys := fmt.Sprintf("%04d", y)
		copy(date[0:4], []byte(ys))
	} else if y >= 10000 {
		// This is probably a bad answer but there isn't really a
		// good answer.
		ys := fmt.Sprintf("%04d", y%1000)
		copy(date[0:4], []byte(ys))
	} else {
		strconv.AppendInt(date[:0], int64(y), 10)
	}
	date[4] = '0' + byte(m/10)
	date[5] = '0' + byte(m%10)
	date[6] = '0' + byte(d/10)
	date[7] = '0' + byte(d%10)
	date[8] = '0' + byte(h/10)
	date[9] = '0' + byte(h%10)
	into = into[:0]
	for _, unit := range q {
		if int(unit) < len(lengthsByQuantum) && lengthsByQuantum[unit] != 0 {
			into = append(into, fullBuf[:l+lengthsByQuantum[unit]])
		}
	}
	return into
}

// viewsByTime returns a list of views for a given timestamp.
func viewsByTime(name string, t time.Time, q TimeQuantum) []string { // nolint: unparam
	y, m, d := t.Date()
	h := t.Hour()
	full := fmt.Sprintf("%s_%04d%02d%02d%02d", name, y, m, d, h)
	l := len(name) + 1
	a := make([]string, 0, len(q))
	for _, unit := range q {
		if int(unit) < len(lengthsByQuantum) && lengthsByQuantum[unit] != 0 {
			a = append(a, full[:l+lengthsByQuantum[unit]])
		}
	}
	return a
}

// viewsByTimeRange returns a list of views to traverse to query a time range.
func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { // nolint: unparam
	t := start

	// Save flags for performance.
	hasYear := q.HasYear()
	hasMonth := q.HasMonth()
	hasDay := q.HasDay()
	hasHour := q.HasHour()

	var results []string

	// Walk up from smallest units to largest units.
	if hasHour || hasDay || hasMonth {
		for t.Before(end) {
			if hasHour {
				if !nextDayGTE(t, end) {
					break
				} else if t.Hour() != 0 {
					results = append(results, viewByTimeUnit(name, t, 'H'))
					t = t.Add(time.Hour)
					continue
				}

			}

			if hasDay {
				if !nextMonthGTE(t, end) {
					break
				} else if t.Day() != 1 {
					results = append(results, viewByTimeUnit(name, t, 'D'))
					t = t.AddDate(0, 0, 1)
					continue
				}
			}

			if hasMonth {
				if !nextYearGTE(t, end) {
					break
				} else if t.Month() != 1 {
					results = append(results, viewByTimeUnit(name, t, 'M'))
					t = addMonth(t)
					continue
				}
			}

			// If a unit exists but isn't set and there are no larger units
			// available then we need to exit the loop because we are no longer
			// making progress.
			break
		}
	}

	// Walk back down from largest units to smallest units.
	for t.Before(end) {
		if hasYear && nextYearGTE(t, end) {
			results = append(results, viewByTimeUnit(name, t, 'Y'))
			t = t.AddDate(1, 0, 0)
		} else if hasMonth && nextMonthGTE(t, end) {
			results = append(results, viewByTimeUnit(name, t, 'M'))
			t = addMonth(t)
		} else if hasDay && nextDayGTE(t, end) {
			results = append(results, viewByTimeUnit(name, t, 'D'))
			t = t.AddDate(0, 0, 1)
		} else if hasHour {
			results = append(results, viewByTimeUnit(name, t, 'H'))
			t = t.Add(time.Hour)
		} else {
			break
		}
	}

	return results
}

// addMonth adds a month similar to time.AddDate(0, 1, 0), but
// in certain edge cases it doesn't normalize for days late in the month.
// In the "YM" case where t.Day is greater than 28, there are
// edge cases where using time.AddDate() to add a month will result
// in two "months" being added (Jan 31 + 1mo = March 2).
func addMonth(t time.Time) time.Time {
	if t.Day() > 28 {
		t = time.Date(t.Year(), t.Month(), 1, t.Hour(), 0, 0, 0, t.Location())
	}
	t = t.AddDate(0, 1, 0)
	return t
}

func nextYearGTE(t time.Time, end time.Time) bool {
	next := t.AddDate(1, 0, 0)
	if next.Year() == end.Year() {
		return true
	}
	return end.After(next)
}

func nextMonthGTE(t time.Time, end time.Time) bool {
	next := t.AddDate(0, 1, 0)
	y1, m1, _ := next.Date()
	y2, m2, _ := end.Date()
	if (y1 == y2) && (m1 == m2) {
		return true
	}
	return end.After(next)
}

func nextDayGTE(t time.Time, end time.Time) bool {
	next := t.AddDate(0, 0, 1)
	y1, m1, d1 := next.Date()
	y2, m2, d2 := end.Date()
	if (y1 == y2) && (m1 == m2) && (d1 == d2) {
		return true
	}
	return end.After(next)
}

// parseTime parses a string or int64 into a time.Time value.
func parseTime(t interface{}) (time.Time, error) {
	var err error
	var calcTime time.Time
	switch v := t.(type) {
	case string:
		if calcTime, err = time.Parse(TimeFormat, v); err != nil {
			// if the default parsing fails, check if user tried to
			// supply partial time eg year and month
			calcTime, err := parsePartialTime(v)
			return calcTime, err
		}
	case int64:
		calcTime = time.Unix(v, 0).UTC()
	default:
		return time.Time{}, errors.New("arg must be a timestamp")
	}
	return calcTime, nil
}

// parsePartialTime parses strings where the time provided is only partial
// eg given 2006-02, it extracts the year and month and the rest of the
// components are set to the default values. The time must have the format
// used in parseTime. The year must be present. The rest of the components are
// optional but if a component is present in the input, this implies that all
// the preceding components are also specified. For example, if the hour is provided
// then the day, month and year must be present. This function could and should be
// simplified
func parsePartialTime(t string) (time.Time, error) {
	// helper parseCustomHourMinute parses strings of the form HH:MM to
	// hour and minute component
	parseHourMinute := func(t string) (hour, minute int, err error) {
		// time should have the format HH:MM
		subStrings := strings.Split(t, ":")
		switch len(subStrings) {
		case 2:
			// has minutes
			minute, err = strconv.Atoi(subStrings[1])
			if err != nil {
				return -1, -1, errors.New("invalid time")
			}
			fallthrough
		case 1:
			hour, err = strconv.Atoi(subStrings[0])
			if err != nil {
				return -1, -1, errors.New("invalid time")
			}
		default:
			return -1, -1, errors.New("invalid time")
		}

		return
	}
	// helper trim function
	trim := func(subMatches []string) (filtered []string, err error) {
		restAreEmpty := func(ss []string) bool {
			for _, s := range ss {
				if s != "" {
					return false
				}
			}
			return true
		}
		if len(subMatches) <= 1 {
			return nil, errors.New("invalid time")
		}
		// ignore full match which is at index 0
		subMatches = subMatches[1:] // ignore full match which is at index 0
		for i, s := range subMatches {
			if s != "" {
				if i > 0 {
					s = s[1:] // remove preceding hyphen or T
				}
				filtered = append(filtered, s)
			} else {
				// rest must be empty for date-time to be valid
				if !restAreEmpty(subMatches[i:]) {
					return nil, errors.New("invalid date-time")
				}
				break
			}
		}
		return filtered, nil
	}

	var errInvalidTime error = errors.New("cannot parse string time")
	var regex = regexp.MustCompile(`^(\d{4})(-\d{2})?(-\d{2})?(T.+)?$`)
	subMatches := regex.FindStringSubmatch(t)
	subMatches, err := trim(subMatches)
	if err != nil {
		return time.Time{}, errInvalidTime
	}
	// defaults
	var (
		yr    int
		month = time.January
		day   = 1
		hour  = 0
		min   = 0
	)
	// year must be set, the rest are optional
	switch len(subMatches) {
	case 4:
		// time
		hour, min, err = parseHourMinute(subMatches[3])
		if err != nil {
			return time.Time{}, errInvalidTime
		}
		fallthrough
	case 3:
		// day
		day, err = strconv.Atoi(subMatches[2])
		if err != nil {
			return time.Time{}, errInvalidTime
		}
		fallthrough
	case 2:
		// month
		monthNum, err := strconv.Atoi(subMatches[1])
		month = time.Month(monthNum)
		if err != nil {
			return time.Time{}, errInvalidTime
		}
		fallthrough
	case 1:
		// year
		yr, err = strconv.Atoi(subMatches[0])
		if err != nil {
			return time.Time{}, errInvalidTime
		}
	default:
		return time.Time{}, errInvalidTime
	}
	return time.Date(yr, month, day, hour, min, 0, 0, time.UTC), nil
}

// minMaxViews returns the min and max view from a list of views
// with a time quantum taken into consideration. It assumes that
// all views represent the same base view name (the logic depends
// on the views sorting correctly in alphabetical order).
func minMaxViews(views []string, q TimeQuantum) (min string, max string) {
	// Sort the list of views.
	sort.Strings(views)

	// Determine the least significant quantum and set that as the
	// number of string characters to compare against.
	var chars int
	if q.HasYear() {
		chars = 4
	} else if q.HasMonth() {
		chars = 6
	} else if q.HasDay() {
		chars = 8
	} else if q.HasHour() {
		chars = 10
	}

	// min: get the first view with the matching number of time chars.
	for _, v := range views {
		if len(viewTimePart(v)) == chars {
			min = v
			break
		}
	}

	// max: get the first view (from the end) with the matching number of time chars.
	for i := len(views) - 1; i >= 0; i-- {
		if len(viewTimePart(views[i])) == chars {
			max = views[i]
			break
		}
	}

	return min, max
}

// timeOfView returns a valid time.Time based on the view string.
// For upper bound use, the result can be adjusted by one by setting
// the `adj` argument to `true`.
func timeOfView(v string, adj bool) (time.Time, error) {
	if v == "" {
		return time.Time{}, nil
	}

	layout := "2006010203"
	timePart := viewTimePart(v)

	switch len(timePart) {
	case 4: // year
		t, err := time.Parse(layout[:4], timePart)
		if err != nil {
			return time.Time{}, err
		}
		if adj {
			t = t.AddDate(1, 0, 0)
		}
		return t, nil
	case 6: // month
		t, err := time.Parse(layout[:6], timePart)
		if err != nil {
			return time.Time{}, err
		}
		if adj {
			t = addMonth(t)
		}
		return t, nil
	case 8: // day
		t, err := time.Parse(layout[:8], timePart)
		if err != nil {
			return time.Time{}, err
		}
		if adj {
			t = t.AddDate(0, 0, 1)
		}
		return t, nil
	case 10: // hour
		t, err := time.Parse(layout[:10], timePart)
		if err != nil {
			return time.Time{}, err
		}
		if adj {
			t = t.Add(time.Hour)
		}
		return t, nil
	}

	return time.Time{}, fmt.Errorf("invalid time format on view: %s", v)
}

// viewTimePart returns the time portion of a string view name.
// e.g. the view "string_201901" would return "201901".
func viewTimePart(v string) string {
	parts := strings.Split(v, "_")
	return parts[len(parts)-1]
}