mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 09:05:55 +00:00
FB-1403 - TTL - Added check for upper bound date (#2053)
* FB-1403 - TTL - Changed TTL to check for the end date of view time * FB-1403 - TTL - clean up unit tests
This commit is contained in:
hphamMolecula
GitHub
parent
dd139cd48d
commit
94e772e6ec
2 changed files with 161 additions and 63 deletions
15
server.go
15
server.go
|
|
@ -870,11 +870,16 @@ func (s *Server) TTLRemoval(ctx context.Context) {
|
|||
if field.Options().Type == "time" {
|
||||
if field.Options().TTL > 0 {
|
||||
for _, view := range field.views() {
|
||||
viewNames := strings.Split(view.name, "_")
|
||||
if len(viewNames) >= 2 {
|
||||
viewTime, err := timeOfView(view.name, false)
|
||||
// view names follow the format of "standard_(time_quantum)"
|
||||
// to get view time, we split the view.name by "_"
|
||||
// then grab the second value (the time quantum)
|
||||
viewName := strings.Split(view.name, "_")
|
||||
if len(viewName) == 2 {
|
||||
// when getting the view time, we want to grab the end date
|
||||
// because start date will aways be older
|
||||
viewTime, err := timeOfView(view.name, true)
|
||||
if err != nil {
|
||||
s.logger.Printf("ttl parse view time: %s", err)
|
||||
s.logger.Printf("view: %s; err: %s", viewName, err)
|
||||
continue
|
||||
}
|
||||
timeSince := time.Since(viewTime)
|
||||
|
|
@ -886,7 +891,7 @@ func (s *Server) TTLRemoval(ctx context.Context) {
|
|||
|
||||
err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), view.name)
|
||||
if err != nil {
|
||||
s.logger.Errorf("ttl delete view: %s", err)
|
||||
s.logger.Errorf("view: %s, ttl delete view: %s", viewName, err)
|
||||
}
|
||||
s.logger.Infof("ttl deleted - index: %s, field: %s, view: %s ", index.name, field.name, view.name)
|
||||
}
|
||||
|
|
|
|||
209
server_test.go
209
server_test.go
|
|
@ -14,7 +14,6 @@ import (
|
|||
)
|
||||
|
||||
func TestTTLRemoval(t *testing.T) {
|
||||
|
||||
cluster := test.MustRunCluster(t, 1)
|
||||
node := cluster.GetNode(0)
|
||||
defer cluster.Close()
|
||||
|
|
@ -23,73 +22,167 @@ func TestTTLRemoval(t *testing.T) {
|
|||
client := node.Client()
|
||||
|
||||
indexName := "i"
|
||||
fieldName := "f"
|
||||
|
||||
// Create indexes and field with ttl lasting 24 hours
|
||||
if err := client.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{TrackExistence: true}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatalf("creating index, err: %v", err)
|
||||
} else if err := client.CreateFieldWithOptions(context.Background(), indexName, fieldName, pilosa.FieldOptions{TTL: time.Hour * 24, Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}); err != nil {
|
||||
t.Fatalf("creating field, err: %v", err)
|
||||
}
|
||||
|
||||
/* Set sample data 1 using this date: '2001-02-03T04:05', this will create these views:
|
||||
- standard
|
||||
- standard_2001
|
||||
- standard_200102
|
||||
- standard_20010203
|
||||
- standard_2001020304
|
||||
Since the sample date here is over 24 hours, all views except "standard" should get deleted
|
||||
*/
|
||||
_, err := client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(1, f=1, 2001-02-03T04:05)"})
|
||||
if err != nil {
|
||||
t.Fatalf("setting sample data 1, err: %v", err)
|
||||
dateNow := time.Now().UTC()
|
||||
dateYesterday := dateNow.Add(-24 * time.Hour)
|
||||
dateCurrentMonth := time.Date(dateNow.Year(), dateNow.Month(), 1, 0, 0, 0, 0, time.UTC)
|
||||
dateLastDayOfMonth := dateCurrentMonth.AddDate(0, 1, 0).Add(-time.Nanosecond)
|
||||
|
||||
var tests = []struct {
|
||||
name string
|
||||
date string
|
||||
expViews []string
|
||||
}{
|
||||
{
|
||||
name: "date_old",
|
||||
date: "2001-02-03T04:05",
|
||||
expViews: []string{"standard"},
|
||||
/* date_old (2001-02-03T04:05), this will create these views:
|
||||
- standard
|
||||
- standard_2001
|
||||
- standard_200102
|
||||
- standard_20010203
|
||||
- standard_2001020304
|
||||
Since the sample date here is over 20 years all views except "standard" should get deleted
|
||||
*/
|
||||
},
|
||||
{
|
||||
name: "date_now",
|
||||
date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour(), dateNow.Minute()),
|
||||
expViews: []string{
|
||||
"standard",
|
||||
"standard_" + fmt.Sprintf("%d", dateNow.Year()),
|
||||
"standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()),
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()),
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour()),
|
||||
},
|
||||
/* For example: current time is 2022-05-11T15:17 (also when the 24 hrs ttl countdown starts) will generate these views:
|
||||
- standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
||||
- standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
||||
- standard_20220511 -> end date is 2022_05_12 T00:00, in future -> keep
|
||||
- standard_2022051115 -> end date is 2022_05_11 T18:00, in future -> keep
|
||||
*/
|
||||
},
|
||||
{
|
||||
name: "date_yesterday",
|
||||
date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour(), dateYesterday.Minute()),
|
||||
expViews: []string{
|
||||
"standard",
|
||||
"standard_" + fmt.Sprintf("%d", dateYesterday.Year()),
|
||||
"standard_" + fmt.Sprintf("%d%02d", dateYesterday.Year(), dateYesterday.Month()),
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day()),
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour()),
|
||||
},
|
||||
/* Example: current time is 2022-05-11T15:17, date_yesterday (2022-05-10T15:17) will generate these views:
|
||||
- standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
||||
- standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
||||
- standard_20220510 -> end date is 2022_05_11 T00:00, within 24 hrs -> keep
|
||||
- standard_2022051015 -> end date is 2022_05_10 T16:00, within 24 hrs -> keep
|
||||
*/
|
||||
},
|
||||
// {
|
||||
// name: "date_first_of_month",
|
||||
// date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateCurrentMonth.Year(), dateCurrentMonth.Month(), 1, 0, 0),
|
||||
// expViews: []string{
|
||||
// "standard",
|
||||
// "standard_" + fmt.Sprintf("%d", dateCurrentMonth.Year()),
|
||||
// "standard_" + fmt.Sprintf("%d%02d", dateCurrentMonth.Year(), dateCurrentMonth.Month()),
|
||||
// },
|
||||
// /* Example: current time is 2022-05-11T15:17, date_first_of_month (2022-05-01T00:00) will generate these views:
|
||||
// - standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
||||
// - standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
||||
// - standard_20220501 -> end date is 2022_05_02 T00:00, older than ttl -> delete
|
||||
// - standard_2022050115 -> end date is 2022_05_01 T01:00, older than ttl -> delete
|
||||
// !!! commenting this test out for now, there is a special case where if today's date is also same as date_first_of_month
|
||||
// in that case, the expected views would have all 4, instead of 2 in the above example, since they are not older than the 24 hr ttl
|
||||
// */
|
||||
// },
|
||||
{
|
||||
name: "date_last_of_month",
|
||||
date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateLastDayOfMonth.Year(), dateLastDayOfMonth.Month(), dateLastDayOfMonth.Day(), dateLastDayOfMonth.Hour(), dateLastDayOfMonth.Minute()),
|
||||
expViews: []string{
|
||||
"standard",
|
||||
"standard_" + fmt.Sprintf("%d", dateLastDayOfMonth.Year()),
|
||||
"standard_" + fmt.Sprintf("%d%02d", dateLastDayOfMonth.Year(), dateLastDayOfMonth.Month()),
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateLastDayOfMonth.Day()),
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateLastDayOfMonth.Day(), dateLastDayOfMonth.Hour()),
|
||||
},
|
||||
/* Example: current time is 2022-05-11T15:17, date_last_of_month (2022-05-31T23:59) will generate these views:
|
||||
- standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
||||
- standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
||||
- standard_20220531 -> end date is 2022_06_01 T00:00, in future -> keep
|
||||
- standard_2022053123 -> end date is 2022_06_01 T00:00, in future -> keep
|
||||
*/
|
||||
},
|
||||
{
|
||||
name: "date_first_hour_day",
|
||||
date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 0, 0),
|
||||
expViews: []string{
|
||||
"standard",
|
||||
"standard_" + fmt.Sprintf("%d", dateNow.Year()),
|
||||
"standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()),
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()),
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 0),
|
||||
},
|
||||
/* Example: current time is 2022-05-11T15:17, date_first_hour_day (2022-05-11T00:00) will generate these views:
|
||||
- standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
||||
- standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
||||
- standard_20220511 -> end date is 2022_05_12 T00:00, in future -> keep
|
||||
- standard_2022051100 -> end date is 2022_05_01 T01:00, within TTL -> keep
|
||||
*/
|
||||
},
|
||||
{
|
||||
name: "date_last_hour_day",
|
||||
date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 23, 59),
|
||||
expViews: []string{
|
||||
"standard",
|
||||
"standard_" + fmt.Sprintf("%d", dateNow.Year()),
|
||||
"standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()),
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()),
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 23),
|
||||
},
|
||||
/* Example: current time is 2022-05-11T15:17, date_last_hour_day (2022-05-11T23:59) will generate these views:
|
||||
- standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
||||
- standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
||||
- standard_20220511 -> end date is 2022_05_12 T00:00, in future -> keep
|
||||
- standard_2022051123 -> end date is 2022_05_12 T00:00, in future-> keep
|
||||
// */
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if err := client.CreateFieldWithOptions(context.Background(), indexName, test.name, pilosa.FieldOptions{TTL: time.Hour * 24, Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}); err != nil {
|
||||
t.Fatalf("creating field, err: %v", err)
|
||||
}
|
||||
|
||||
dateNow := time.Now()
|
||||
dateNowString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour(), dateNow.Minute())
|
||||
/* Set sample data 2 using current time.
|
||||
For example: current time is 2022-03-03T15:17 (also when the 24 hrs ttl countdown starts) will generate these views:
|
||||
- standard_2022 -> gets converted to 2022_01_01, over 24 hours for ttl -> deleted
|
||||
- standard_202203 -> gets converted to 2022_03_01, over 24 hours for ttl -> deleted
|
||||
- standard_20220303 -> gets converted to 2022_03_03, within 24 hours -> keep
|
||||
- standard_2022030315 -> gets converted to 2022_03_03 15:00, within 24 hours -> keep
|
||||
*/
|
||||
_, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(2, f=2, " + dateNowString + ")"})
|
||||
if err != nil {
|
||||
t.Fatalf("setting sample data 2, err: %v", err)
|
||||
}
|
||||
// set data
|
||||
_, err := client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(1, " + test.name + "=1, " + test.date + ")"})
|
||||
if err != nil {
|
||||
t.Fatalf("setting sample data, err: %v", err)
|
||||
}
|
||||
|
||||
/* Set sample data 3 using yesterday's date
|
||||
All views generated from this date should be deleted
|
||||
*/
|
||||
dateYesterday := time.Now().Add(-24*time.Hour + -1*time.Nanosecond)
|
||||
dateYesterdayString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour(), dateYesterday.Minute())
|
||||
_, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(3, f=3, " + dateYesterdayString + ")"})
|
||||
if err != nil {
|
||||
t.Fatalf("setting sample data 3, err: %v", err)
|
||||
}
|
||||
// run TTLRemoval
|
||||
node.Server.TTLRemoval(context.Background())
|
||||
|
||||
node.Server.TTLRemoval(context.Background())
|
||||
// Get all the views for given index + field
|
||||
views, err := node.API.Views(context.Background(), indexName, test.name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var viewNames []string
|
||||
for _, view := range views {
|
||||
viewNames = append(viewNames, view.Name())
|
||||
}
|
||||
sort.Strings(viewNames)
|
||||
|
||||
// Get all the views for given index + field
|
||||
views, err := node.API.Views(context.Background(), indexName, fieldName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expectedViewNames := []string{
|
||||
"standard",
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()),
|
||||
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour()),
|
||||
}
|
||||
|
||||
var viewNames []string
|
||||
for _, view := range views {
|
||||
viewNames = append(viewNames, view.Name())
|
||||
}
|
||||
sort.Strings(viewNames)
|
||||
|
||||
if !reflect.DeepEqual(expectedViewNames, viewNames) {
|
||||
t.Fatalf("after ttl removal, expected %v, but got %v", expectedViewNames, viewNames)
|
||||
if !reflect.DeepEqual(test.expViews, viewNames) {
|
||||
t.Fatalf("after ttl removal, expected %v, but got %v", test.expViews, viewNames)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue