FB-1456 - TTL - fixed views not returning correct results (#2062)

* FB- 1456 - TTL - fixed views not returning correct results when least precise quantum are deleted

* FB-1456 - TTL - PR - fixed comment

* FB-1456 - TTL - changed getQuantum to getLowestGranularityQuantum since we only care about the least precise quantum that is available

* FB-1456 - TTL - removed unit test used for debug

* FB-1456 - TTL - fixed comments

* [FB-1435] BSI Base Fix (#2056)

* add bsi base back to int value

* test bsi base/min/max for IntFields

motivated by bsi base not being added back to values
in extract calls when min was a positive integer.

* FB-1456 - TTL - fixed comments

Co-authored-by: Samir Patel <48686912+54mir@users.noreply.github.com>

server_test.go

@@ -150,8 +150,8 @@ func TestTTLRemoval(t *testing.T) {
 			- standard_2022			-> end date is 2023_01_01 T00:00, in future -> keep
 			- standard_202205		-> end date is 2022_06_01 T00:00, in future -> keep
 			- standard_20220511		-> end date is 2022_05_12 T00:00, in future -> keep
-			- standard_2022051123	-> end date is 2022_05_12 T00:00, in future-> keep
-			// */
+			- standard_2022051123	-> end date is 2022_05_12 T00:00, in future -> keep
+			*/
 		},
 	}
 	for _, test := range tests {

time.go

@@ -29,6 +29,9 @@ func (q TimeQuantum) HasDay() bool { return strings.ContainsRune(string(q), 'D')
 // HasHour returns true if the quantum contains a 'H' unit.
 func (q TimeQuantum) HasHour() bool { return strings.ContainsRune(string(q), 'H') }
 
+// IsEmpty returns true if the quantum is empty.
+func (q TimeQuantum) IsEmpty() bool { return string(q) == "" }
+
 func (q TimeQuantum) Granularity() rune {
 	var g rune
 	for _, g = range q {
@@ -410,6 +413,28 @@ func minMaxViews(views []string, q TimeQuantum) (min string, max string) {
 	// Sort the list of views.
 	sort.Strings(views)
 
+	// get the lowest granularity quantum available from the given views
+	lowestQuantumFromViews := getLowestGranularityQuantum(views)
+
+	/*
+		- lowestQuantumFromViews was added because of a unique case where the view for least precise quantum was somehow deleted:
+			- ex: q="YMDH" but views only have "DH" (ex: std_20220531, std_2022053123)
+			- without lowestQuantumFromViews, this function would return empty for min, max (as if there were no time views)
+				because it would look for "Y" views but cant find it since it was deleted
+			- with lowestQuantumFromViews, this function will look at the available quantum from views (ex: "DH")
+				grab the least precise quantum ("D")
+				and use D (day) view (ex: std_20220531)
+		- use lowestQuantumFromViews if
+		1. lowestQuantumFromViews is not empty
+		2. lowestQuantumFromViews is actually a substring of q
+		 - lowestQuantumFromViews has to be a substring of q because if q="Y" but views quantum="DH" (ex: std_20220531, std_2022053123),
+		 the "DH" quantum wont matter since originally q of "Y" didnt include "DH"
+	*/
+
+	if !(lowestQuantumFromViews.IsEmpty()) && strings.Contains(q.String(), lowestQuantumFromViews.String()) {
+		q = lowestQuantumFromViews
+	}
+
 	// Determine the least precise quantum and set that as the
 	// number of string characters to compare against.
 	var chars int
@@ -505,3 +530,57 @@ func viewTimePart(v string) string {
 	}
 	return parts[len(parts)-1]
 }
+
+// getLowestGranularityQuantum returns lowest granularity quantum from a list of views
+// e.g.
+//	[std_2001, std_200102, std_20010203, std_2001020304] - returns "Y" since year is the lowest granularity
+//	[std_2001020304, std_200102, std_20010203] - returns "M", the order of views should not affect lowest granularity
+func getLowestGranularityQuantum(views []string) TimeQuantum {
+
+	// Time quantum with the highest level of granularity we support
+	timeQuantum := "YMDH"
+
+	write_Y := false
+	write_M := false
+	write_D := false
+	write_H := false
+	for _, v := range views {
+		viewTime := viewTimePart(v)
+		if viewTime != "" {
+			if len(viewTime) == 4 {
+				if !write_Y {
+					write_Y = true
+				}
+			} else if len(viewTime) == 6 {
+				if !write_M {
+					write_M = true
+				}
+			} else if len(viewTime) == 8 {
+				if !write_D {
+					write_D = true
+				}
+			} else if len(viewTime) == 10 {
+				if !write_H {
+					write_H = true
+				}
+			}
+		}
+	}
+
+	lowestGranularity := ""
+	if write_Y {
+		// Y
+		lowestGranularity = timeQuantum[:1]
+	} else if !write_Y && write_M {
+		// M
+		lowestGranularity = timeQuantum[1:2]
+	} else if !write_Y && !write_M && write_D {
+		// D
+		lowestGranularity = timeQuantum[2:3]
+	} else if !write_Y && !write_M && !write_D && write_H {
+		// H
+		lowestGranularity = timeQuantum[3:4]
+	}
+
+	return TimeQuantum(lowestGranularity)
+}

time_internal_test.go

@@ -204,6 +204,13 @@ func TestMinMaxViews(t *testing.T) {
 				"std_2019",
 				"std_2022",
 			},
+			{
+				// unordered views should not affect the results
+				[]string{"std_202002", "std_2022073123", "std_2020", "std_202002", "std_2022", "std_2019", "std_2022063023"},
+				mustParseTimeQuantum("Y"),
+				"std_2019",
+				"std_2022",
+			},
 			{
 				[]string{"std_201902", "std_201901"},
 				mustParseTimeQuantum("M"),
@@ -228,6 +235,27 @@ func TestMinMaxViews(t *testing.T) {
 				"",
 				"",
 			},
+			{
+				// quantum is YMDH, views only have "DH" views
+				[]string{"std_20220531", "std_2022063023", "std_20220731", "std_2022073123"},
+				mustParseTimeQuantum("YMDH"),
+				"std_20220531",
+				"std_20220731",
+			},
+			{
+				// quantum is Y, views have D,H but dont have Y views
+				[]string{"std_20220531", "std_2022053123"},
+				mustParseTimeQuantum("Y"),
+				"",
+				"",
+			},
+			{
+				// quantum is M, views ignore Y view, only look at M view
+				[]string{"std_2022", "std_202205", "std_20220531", "std_2022053123"},
+				mustParseTimeQuantum("M"),
+				"std_202205",
+				"std_202205",
+			},
 		}
 		for i, test := range tests {
 			if min, max := minMaxViews(test.views, test.q); min != test.min {
@@ -410,3 +438,77 @@ func TestViewTimePart(t *testing.T) {
 		}
 	}
 }
+
+func TestGetLowestGranularityQuantum(t *testing.T) {
+	tests := []struct {
+		name       string
+		views      []string
+		expQuantum TimeQuantum
+	}{
+		{
+			name:       "Y",
+			views:      []string{"std_2022", "std_202205", "std_20220531", "std_2022053123"},
+			expQuantum: TimeQuantum("Y"),
+		},
+		{
+			name:       "empty",
+			views:      []string{},
+			expQuantum: TimeQuantum(""),
+		},
+		{
+			name:       "empty, not number",
+			views:      []string{"std_abc"},
+			expQuantum: TimeQuantum(""),
+		},
+		{
+			name:       "duplicate",
+			views:      []string{"std_2022", "std_202205", "std_20220531", "std_2022053123", "std_2022", "std_202205", "std_20220531", "std_2022053123"},
+			expQuantum: TimeQuantum("Y"),
+		},
+		{
+			name:       "only Y",
+			views:      []string{"std_2022"},
+			expQuantum: TimeQuantum("Y"),
+		},
+		{
+			name:       "only M",
+			views:      []string{"std_202205"},
+			expQuantum: TimeQuantum("M"),
+		},
+		{
+			name:       "only D",
+			views:      []string{"std_20220531"},
+			expQuantum: TimeQuantum("D"),
+		},
+		{
+			name:       "only H",
+			views:      []string{"std_2022053123"},
+			expQuantum: TimeQuantum("H"),
+		},
+		{
+			name:       "Y unordered",
+			views:      []string{"std_202205", "std_20220531", "std_2022053123", "std_2022"},
+			expQuantum: TimeQuantum("Y"),
+		},
+		{
+			name:       "M unordered",
+			views:      []string{"std_2022053123", "std_202205", "std_20220531"},
+			expQuantum: TimeQuantum("M"),
+		},
+		{
+			name:       "D unordered",
+			views:      []string{"std_2022053123", "std_20220531"},
+			expQuantum: TimeQuantum("D"),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			quantum := getLowestGranularityQuantum(test.views)
+
+			if quantum.String() != test.expQuantum.String() {
+				t.Errorf("expected field: '%v', got: '%v'", test.expQuantum.String(), quantum.String())
+			}
+		})
+	}
+}