featurebase/time.go

// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa

import (
	"errors"
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

// ErrInvalidTimeQuantum is returned when parsing a time quantum.
var ErrInvalidTimeQuantum = errors.New("invalid time quantum")

// TimeQuantum represents a time granularity for time-based bitmaps.
type TimeQuantum string

// HasYear returns true if the quantum contains a 'Y' unit.
func (q TimeQuantum) HasYear() bool { return strings.ContainsRune(string(q), 'Y') }

// HasMonth returns true if the quantum contains a 'M' unit.
func (q TimeQuantum) HasMonth() bool { return strings.ContainsRune(string(q), 'M') }

// HasDay returns true if the quantum contains a 'D' unit.
func (q TimeQuantum) HasDay() bool { return strings.ContainsRune(string(q), 'D') }

// HasHour returns true if the quantum contains a 'H' unit.
func (q TimeQuantum) HasHour() bool { return strings.ContainsRune(string(q), 'H') }

// IsEmpty returns true if the quantum is empty.
func (q TimeQuantum) IsEmpty() bool { return string(q) == "" }

func (q TimeQuantum) Granularity() rune {
	var g rune
	for _, g = range q {
	}
	return g
}

// Valid returns true if q is a valid time quantum value.
func (q TimeQuantum) Valid() bool {
	switch q {
	case "Y", "YM", "YMD", "YMDH",
		"M", "MD", "MDH",
		"D", "DH",
		"H",
		"":
		return true
	default:
		return false
	}
}

// The following methods are required to implement pflag Value interface.

// Set sets the time quantum value.
func (q *TimeQuantum) Set(value string) error {
	*q = TimeQuantum(value)
	return nil
}

func (q TimeQuantum) String() string {
	return string(q)
}

// Type returns the type of a time quantum value.
func (q TimeQuantum) Type() string {
	return "TimeQuantum"
}

// viewByTimeUnit returns the view name for time with a given quantum unit.
func viewByTimeUnit(name string, t time.Time, unit rune) string {
	switch unit {
	case 'Y':
		return fmt.Sprintf("%s_%s", name, t.Format("2006"))
	case 'M':
		return fmt.Sprintf("%s_%s", name, t.Format("200601"))
	case 'D':
		return fmt.Sprintf("%s_%s", name, t.Format("20060102"))
	case 'H':
		return fmt.Sprintf("%s_%s", name, t.Format("2006010215"))
	default:
		return ""
	}
}

// YYYYMMDDHH lengths. Note that this is a []int, not a map[byte]int, so
// the lookups can be cheaper.
var lengthsByQuantum = []int{
	'Y': 4,
	'M': 6,
	'D': 8,
	'H': 10,
}

// viewsByTimeInto computes the list of views for a given time. It expects
// to be given an initial buffer of the form `name_YYYYMMDDHH`, and a slice
// of []bytes. This allows us to reuse the buffer for all the sub-buffers,
// and also to reuse the slice of slices, to eliminate all those allocations.
// This might seem crazy, but even including the JSON parsing and all the
// disk activity, the straightforward viewsByTime implementation was 25%
// of runtime in an ingest test.
func viewsByTimeInto(fullBuf []byte, into [][]byte, t time.Time, q TimeQuantum) [][]byte {
	l := len(fullBuf) - 10
	date := fullBuf[l : l+10]
	y, m, d := t.Date()
	h := t.Hour()
	// Did you know that Sprintf, Printf, and other things like that all
	// do allocations, and that doing allocations in a tight loop like this
	// is stunningly expensive? viewsByTime was 25% of an ingest test's
	// total CPU, not counting the garbage collector overhead. This is about
	// 3%. No, I'm not totally sure that justifies it.
	if y < 1000 {
		ys := fmt.Sprintf("%04d", y)
		copy(date[0:4], []byte(ys))
	} else if y >= 10000 {
		// This is probably a bad answer but there isn't really a
		// good answer.
		ys := fmt.Sprintf("%04d", y%1000)
		copy(date[0:4], []byte(ys))
	} else {
		strconv.AppendInt(date[:0], int64(y), 10)
	}
	date[4] = '0' + byte(m/10)
	date[5] = '0' + byte(m%10)
	date[6] = '0' + byte(d/10)
	date[7] = '0' + byte(d%10)
	date[8] = '0' + byte(h/10)
	date[9] = '0' + byte(h%10)
	into = into[:0]
	for _, unit := range q {
		if int(unit) < len(lengthsByQuantum) && lengthsByQuantum[unit] != 0 {
			into = append(into, fullBuf[:l+lengthsByQuantum[unit]])
		}
	}
	return into
}

// viewsByTime returns a list of views for a given timestamp.
func viewsByTime(name string, t time.Time, q TimeQuantum) []string { // nolint: unparam
	y, m, d := t.Date()
	h := t.Hour()
	full := fmt.Sprintf("%s_%04d%02d%02d%02d", name, y, m, d, h)
	l := len(name) + 1
	a := make([]string, 0, len(q))
	for _, unit := range q {
		if int(unit) < len(lengthsByQuantum) && lengthsByQuantum[unit] != 0 {
			a = append(a, full[:l+lengthsByQuantum[unit]])
		}
	}
	return a
}

// viewsByTimeRange returns a list of views to traverse to query a time range.
func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { // nolint: unparam
	t := start

	// Save flags for performance.
	hasYear := q.HasYear()
	hasMonth := q.HasMonth()
	hasDay := q.HasDay()
	hasHour := q.HasHour()

	var results []string

	// Walk up from smallest units to largest units.
	if hasHour || hasDay || hasMonth {
		for t.Before(end) {
			if hasHour {
				if !nextDayGTE(t, end) {
					break
				} else if t.Hour() != 0 {
					results = append(results, viewByTimeUnit(name, t, 'H'))
					t = t.Add(time.Hour)
					continue
				}

			}

			if hasDay {
				if !nextMonthGTE(t, end) {
					break
				} else if t.Day() != 1 {
					results = append(results, viewByTimeUnit(name, t, 'D'))
					t = t.AddDate(0, 0, 1)
					continue
				}
			}

			if hasMonth {
				if !nextYearGTE(t, end) {
					break
				} else if t.Month() != 1 {
					results = append(results, viewByTimeUnit(name, t, 'M'))
					t = addMonth(t)
					continue
				}
			}

			// If a unit exists but isn't set and there are no larger units
			// available then we need to exit the loop because we are no longer
			// making progress.
			break
		}
	}

	// Walk back down from largest units to smallest units.
	for t.Before(end) {
		if hasYear && nextYearGTE(t, end) {
			results = append(results, viewByTimeUnit(name, t, 'Y'))
			t = t.AddDate(1, 0, 0)
		} else if hasMonth && nextMonthGTE(t, end) {
			results = append(results, viewByTimeUnit(name, t, 'M'))
			t = addMonth(t)
		} else if hasDay && nextDayGTE(t, end) {
			results = append(results, viewByTimeUnit(name, t, 'D'))
			t = t.AddDate(0, 0, 1)
		} else if hasHour {
			results = append(results, viewByTimeUnit(name, t, 'H'))
			t = t.Add(time.Hour)
		} else {
			break
		}
	}

	return results
}

// addMonth adds a month similar to time.AddDate(0, 1, 0), but
// in certain edge cases it doesn't normalize for days late in the month.
// In the "YM" case where t.Day is greater than 28, there are
// edge cases where using time.AddDate() to add a month will result
// in two "months" being added (Jan 31 + 1mo = March 2).
func addMonth(t time.Time) time.Time {
	if t.Day() > 28 {
		t = time.Date(t.Year(), t.Month(), 1, t.Hour(), 0, 0, 0, t.Location())
	}
	t = t.AddDate(0, 1, 0)
	return t
}

func nextYearGTE(t time.Time, end time.Time) bool {
	next := t.AddDate(1, 0, 0)
	if next.Year() == end.Year() {
		return true
	}
	return end.After(next)
}

func nextMonthGTE(t time.Time, end time.Time) bool {
	next := t.AddDate(0, 1, 0)
	y1, m1, _ := next.Date()
	y2, m2, _ := end.Date()
	if (y1 == y2) && (m1 == m2) {
		return true
	}
	return end.After(next)
}

func nextDayGTE(t time.Time, end time.Time) bool {
	next := t.AddDate(0, 0, 1)
	y1, m1, d1 := next.Date()
	y2, m2, d2 := end.Date()
	if (y1 == y2) && (m1 == m2) && (d1 == d2) {
		return true
	}
	return end.After(next)
}

// parseTime parses a string or int64 into a time.Time value.
func parseTime(t interface{}) (time.Time, error) {
	var err error
	var calcTime time.Time
	switch v := t.(type) {
	case string:
		if calcTime, err = time.Parse(TimeFormat, v); err != nil {
			// if the default parsing fails, check if user tried to
			// supply partial time eg year and month
			calcTime, err = parsePartialTime(v)
			return calcTime, err
		}
	case int64:
		calcTime = time.Unix(v, 0).UTC()
	default:
		return time.Time{}, errors.New("arg must be a timestamp")
	}
	return calcTime, nil
}

// parsePartialTime parses strings where the time provided is only partial
// eg given 2006-02, it extracts the year and month and the rest of the
// components are set to the default values. The time must have the format
// used in parseTime. The year must be present. The rest of the components are
// optional but if a component is present in the input, this implies that all
// the preceding components are also specified. For example, if the hour is provided
// then the day, month and year must be present. This function could and should be
// simplified
func parsePartialTime(t string) (time.Time, error) {
	// helper parseCustomHourMinute parses strings of the form HH:MM to
	// hour and minute component
	parseHourMinute := func(t string) (hour, minute int, err error) {
		// time should have the format HH:MM
		subStrings := strings.Split(t, ":")
		switch len(subStrings) {
		case 2:
			// has minutes
			minute, err = strconv.Atoi(subStrings[1])
			if err != nil {
				return -1, -1, errors.New("invalid time")
			}
			fallthrough
		case 1:
			hour, err = strconv.Atoi(subStrings[0])
			if err != nil {
				return -1, -1, errors.New("invalid time")
			}
		default:
			return -1, -1, errors.New("invalid time")
		}

		return
	}
	// helper trim function
	trim := func(subMatches []string) (filtered []string, err error) {
		restAreEmpty := func(ss []string) bool {
			for _, s := range ss {
				if s != "" {
					return false
				}
			}
			return true
		}
		if len(subMatches) <= 1 {
			return nil, errors.New("invalid time")
		}
		// ignore full match which is at index 0
		subMatches = subMatches[1:] // ignore full match which is at index 0
		for i, s := range subMatches {
			if s != "" {
				if i > 0 {
					s = s[1:] // remove preceding hyphen or T
				}
				filtered = append(filtered, s)
			} else {
				// rest must be empty for date-time to be valid
				if !restAreEmpty(subMatches[i:]) {
					return nil, errors.New("invalid date-time")
				}
				break
			}
		}
		return filtered, nil
	}

	var errInvalidTime error = errors.New("cannot parse string time")
	var regex = regexp.MustCompile(`^(\d{4})(-\d{2})?(-\d{2})?(T.+)?$`)
	subMatches := regex.FindStringSubmatch(t)
	subMatches, err := trim(subMatches)
	if err != nil {
		return time.Time{}, errInvalidTime
	}
	// defaults
	var (
		yr    int
		month = time.January
		day   = 1
		hour  = 0
		min   = 0
	)
	// year must be set, the rest are optional
	switch len(subMatches) {
	case 4:
		// time
		hour, min, err = parseHourMinute(subMatches[3])
		if err != nil {
			return time.Time{}, errInvalidTime
		}
		fallthrough
	case 3:
		// day
		day, err = strconv.Atoi(subMatches[2])
		if err != nil {
			return time.Time{}, errInvalidTime
		}
		fallthrough
	case 2:
		// month
		monthNum, err := strconv.Atoi(subMatches[1])
		month = time.Month(monthNum)
		if err != nil {
			return time.Time{}, errInvalidTime
		}
		fallthrough
	case 1:
		// year
		yr, err = strconv.Atoi(subMatches[0])
		if err != nil {
			return time.Time{}, errInvalidTime
		}
	default:
		return time.Time{}, errInvalidTime
	}
	return time.Date(yr, month, day, hour, min, 0, 0, time.UTC), nil
}

// minMaxViews returns the min and max view from a list of views
// with a time quantum taken into consideration. It assumes that
// all views represent the same base view name (the logic depends
// on the views sorting correctly in alphabetical order).
func minMaxViews(views []string, q TimeQuantum) (min string, max string) {
	// Sort the list of views.
	sort.Strings(views)

	// get the lowest granularity quantum available from the given views
	lowestQuantumFromViews := getLowestGranularityQuantum(views)

	/*
		- lowestQuantumFromViews was added because of a unique case where the view for least precise quantum was somehow deleted:
			- ex: q="YMDH" but views only have "DH" (ex: std_20220531, std_2022053123)
			- without lowestQuantumFromViews, this function would return empty for min, max (as if there were no time views)
				because it would look for "Y" views but cant find it since it was deleted
			- with lowestQuantumFromViews, this function will look at the available quantum from views (ex: "DH")
				grab the least precise quantum ("D")
				and use D (day) view (ex: std_20220531)
		- use lowestQuantumFromViews if
		1. lowestQuantumFromViews is not empty
		2. lowestQuantumFromViews is actually a substring of q
		 - lowestQuantumFromViews has to be a substring of q because if q="Y" but views quantum="DH" (ex: std_20220531, std_2022053123),
		 the "DH" quantum wont matter since originally q of "Y" didnt include "DH"
	*/

	if !(lowestQuantumFromViews.IsEmpty()) && strings.Contains(q.String(), lowestQuantumFromViews.String()) {
		q = lowestQuantumFromViews
	}

	// Determine the least precise quantum and set that as the
	// number of string characters to compare against.
	var chars int
	if q.HasYear() {
		chars = 4
	} else if q.HasMonth() {
		chars = 6
	} else if q.HasDay() {
		chars = 8
	} else if q.HasHour() {
		chars = 10
	}

	// min: get the first view with the matching number of time chars.
	for _, v := range views {
		if len(viewTimePart(v)) == chars {
			min = v
			break
		}
	}

	// max: get the first view (from the end) with the matching number of time chars.
	for i := len(views) - 1; i >= 0; i-- {
		if len(viewTimePart(views[i])) == chars {
			max = views[i]
			break
		}
	}

	return min, max
}

// timeOfView returns a valid time.Time based on the view string.
// For upper bound use, the result can be adjusted by one by setting
// the `adj` argument to `true`.
func timeOfView(v string, adj bool) (time.Time, error) {
	if v == "" {
		return time.Time{}, nil
	}

	layout := "2006010215"
	timePart := viewTimePart(v)

	switch len(timePart) {
	case 4: // year
		t, err := time.Parse(layout[:4], timePart)
		if err != nil {
			return time.Time{}, err
		}
		if adj {
			t = t.AddDate(1, 0, 0)
		}
		return t, nil
	case 6: // month
		t, err := time.Parse(layout[:6], timePart)
		if err != nil {
			return time.Time{}, err
		}
		if adj {
			t = addMonth(t)
		}
		return t, nil
	case 8: // day
		t, err := time.Parse(layout[:8], timePart)
		if err != nil {
			return time.Time{}, err
		}
		if adj {
			t = t.AddDate(0, 0, 1)
		}
		return t, nil
	case 10: // hour
		t, err := time.Parse(layout[:10], timePart)
		if err != nil {
			return time.Time{}, err
		}
		if adj {
			t = t.Add(time.Hour)
		}
		return t, nil
	}

	return time.Time{}, fmt.Errorf("invalid time format on view: %s", v)
}

// viewTimePart returns the time portion of a string view name.
// e.g. the view "string_201901" would return "201901".
func viewTimePart(v string) string {
	parts := strings.Split(v, "_")
	if _, err := strconv.Atoi(parts[len(parts)-1]); err != nil {
		// it's not a number!
		return ""
	}
	return parts[len(parts)-1]
}

// getLowestGranularityQuantum returns lowest granularity quantum from a list of views
// e.g.
//
//	[std_2001, std_200102, std_20010203, std_2001020304] - returns "Y" since year is the lowest granularity
//	[std_2001020304, std_200102, std_20010203] - returns "M", the order of views should not affect lowest granularity
func getLowestGranularityQuantum(views []string) TimeQuantum {

	// Time quantum with the highest level of granularity we support
	timeQuantum := "YMDH"

	write_Y := false
	write_M := false
	write_D := false
	write_H := false
	for _, v := range views {
		viewTime := viewTimePart(v)
		if viewTime != "" {
			if len(viewTime) == 4 {
				if !write_Y {
					write_Y = true
				}
			} else if len(viewTime) == 6 {
				if !write_M {
					write_M = true
				}
			} else if len(viewTime) == 8 {
				if !write_D {
					write_D = true
				}
			} else if len(viewTime) == 10 {
				if !write_H {
					write_H = true
				}
			}
		}
	}

	lowestGranularity := ""
	if write_Y {
		// Y
		lowestGranularity = timeQuantum[:1]
	} else if !write_Y && write_M {
		// M
		lowestGranularity = timeQuantum[1:2]
	} else if !write_Y && !write_M && write_D {
		// D
		lowestGranularity = timeQuantum[2:3]
	} else if !write_Y && !write_M && !write_D && write_H {
		// H
		lowestGranularity = timeQuantum[3:4]
	}

	return TimeQuantum(lowestGranularity)
}