From 94e772e6ecf34f91534e720d70aa6cc48441c90e Mon Sep 17 00:00:00 2001 From: hphamMolecula <89804723+hphamMolecula@users.noreply.github.com> Date: Wed, 11 May 2022 13:57:29 -0500 Subject: [PATCH] FB-1403 - TTL - Added check for upper bound date (#2053) * FB-1403 - TTL - Changed TTL to check for the end date of view time * FB-1403 - TTL - clean up unit tests --- server.go | 15 ++-- server_test.go | 209 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 161 insertions(+), 63 deletions(-) diff --git a/server.go b/server.go index 354c3584b..baa5f8968 100644 --- a/server.go +++ b/server.go @@ -870,11 +870,16 @@ func (s *Server) TTLRemoval(ctx context.Context) { if field.Options().Type == "time" { if field.Options().TTL > 0 { for _, view := range field.views() { - viewNames := strings.Split(view.name, "_") - if len(viewNames) >= 2 { - viewTime, err := timeOfView(view.name, false) + // view names follow the format of "standard_(time_quantum)" + // to get view time, we split the view.name by "_" + // then grab the second value (the time quantum) + viewName := strings.Split(view.name, "_") + if len(viewName) == 2 { + // when getting the view time, we want to grab the end date + // because start date will aways be older + viewTime, err := timeOfView(view.name, true) if err != nil { - s.logger.Printf("ttl parse view time: %s", err) + s.logger.Printf("view: %s; err: %s", viewName, err) continue } timeSince := time.Since(viewTime) @@ -886,7 +891,7 @@ func (s *Server) TTLRemoval(ctx context.Context) { err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), view.name) if err != nil { - s.logger.Errorf("ttl delete view: %s", err) + s.logger.Errorf("view: %s, ttl delete view: %s", viewName, err) } s.logger.Infof("ttl deleted - index: %s, field: %s, view: %s ", index.name, field.name, view.name) } diff --git a/server_test.go b/server_test.go index 1e0b28ae9..1c82d629e 100644 --- a/server_test.go +++ b/server_test.go @@ -14,7 +14,6 @@ import ( ) func TestTTLRemoval(t *testing.T) { - cluster := test.MustRunCluster(t, 1) node := cluster.GetNode(0) defer cluster.Close() @@ -23,73 +22,167 @@ func TestTTLRemoval(t *testing.T) { client := node.Client() indexName := "i" - fieldName := "f" // Create indexes and field with ttl lasting 24 hours if err := client.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{TrackExistence: true}); err != nil && err != pilosa.ErrIndexExists { t.Fatalf("creating index, err: %v", err) - } else if err := client.CreateFieldWithOptions(context.Background(), indexName, fieldName, pilosa.FieldOptions{TTL: time.Hour * 24, Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}); err != nil { - t.Fatalf("creating field, err: %v", err) } - /* Set sample data 1 using this date: '2001-02-03T04:05', this will create these views: - - standard - - standard_2001 - - standard_200102 - - standard_20010203 - - standard_2001020304 - Since the sample date here is over 24 hours, all views except "standard" should get deleted - */ - _, err := client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(1, f=1, 2001-02-03T04:05)"}) - if err != nil { - t.Fatalf("setting sample data 1, err: %v", err) + dateNow := time.Now().UTC() + dateYesterday := dateNow.Add(-24 * time.Hour) + dateCurrentMonth := time.Date(dateNow.Year(), dateNow.Month(), 1, 0, 0, 0, 0, time.UTC) + dateLastDayOfMonth := dateCurrentMonth.AddDate(0, 1, 0).Add(-time.Nanosecond) + + var tests = []struct { + name string + date string + expViews []string + }{ + { + name: "date_old", + date: "2001-02-03T04:05", + expViews: []string{"standard"}, + /* date_old (2001-02-03T04:05), this will create these views: + - standard + - standard_2001 + - standard_200102 + - standard_20010203 + - standard_2001020304 + Since the sample date here is over 20 years all views except "standard" should get deleted + */ + }, + { + name: "date_now", + date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour(), dateNow.Minute()), + expViews: []string{ + "standard", + "standard_" + fmt.Sprintf("%d", dateNow.Year()), + "standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()), + "standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()), + "standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour()), + }, + /* For example: current time is 2022-05-11T15:17 (also when the 24 hrs ttl countdown starts) will generate these views: + - standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep + - standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep + - standard_20220511 -> end date is 2022_05_12 T00:00, in future -> keep + - standard_2022051115 -> end date is 2022_05_11 T18:00, in future -> keep + */ + }, + { + name: "date_yesterday", + date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour(), dateYesterday.Minute()), + expViews: []string{ + "standard", + "standard_" + fmt.Sprintf("%d", dateYesterday.Year()), + "standard_" + fmt.Sprintf("%d%02d", dateYesterday.Year(), dateYesterday.Month()), + "standard_" + fmt.Sprintf("%d%02d%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day()), + "standard_" + fmt.Sprintf("%d%02d%02d%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour()), + }, + /* Example: current time is 2022-05-11T15:17, date_yesterday (2022-05-10T15:17) will generate these views: + - standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep + - standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep + - standard_20220510 -> end date is 2022_05_11 T00:00, within 24 hrs -> keep + - standard_2022051015 -> end date is 2022_05_10 T16:00, within 24 hrs -> keep + */ + }, + // { + // name: "date_first_of_month", + // date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateCurrentMonth.Year(), dateCurrentMonth.Month(), 1, 0, 0), + // expViews: []string{ + // "standard", + // "standard_" + fmt.Sprintf("%d", dateCurrentMonth.Year()), + // "standard_" + fmt.Sprintf("%d%02d", dateCurrentMonth.Year(), dateCurrentMonth.Month()), + // }, + // /* Example: current time is 2022-05-11T15:17, date_first_of_month (2022-05-01T00:00) will generate these views: + // - standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep + // - standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep + // - standard_20220501 -> end date is 2022_05_02 T00:00, older than ttl -> delete + // - standard_2022050115 -> end date is 2022_05_01 T01:00, older than ttl -> delete + // !!! commenting this test out for now, there is a special case where if today's date is also same as date_first_of_month + // in that case, the expected views would have all 4, instead of 2 in the above example, since they are not older than the 24 hr ttl + // */ + // }, + { + name: "date_last_of_month", + date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateLastDayOfMonth.Year(), dateLastDayOfMonth.Month(), dateLastDayOfMonth.Day(), dateLastDayOfMonth.Hour(), dateLastDayOfMonth.Minute()), + expViews: []string{ + "standard", + "standard_" + fmt.Sprintf("%d", dateLastDayOfMonth.Year()), + "standard_" + fmt.Sprintf("%d%02d", dateLastDayOfMonth.Year(), dateLastDayOfMonth.Month()), + "standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateLastDayOfMonth.Day()), + "standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateLastDayOfMonth.Day(), dateLastDayOfMonth.Hour()), + }, + /* Example: current time is 2022-05-11T15:17, date_last_of_month (2022-05-31T23:59) will generate these views: + - standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep + - standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep + - standard_20220531 -> end date is 2022_06_01 T00:00, in future -> keep + - standard_2022053123 -> end date is 2022_06_01 T00:00, in future -> keep + */ + }, + { + name: "date_first_hour_day", + date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 0, 0), + expViews: []string{ + "standard", + "standard_" + fmt.Sprintf("%d", dateNow.Year()), + "standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()), + "standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()), + "standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 0), + }, + /* Example: current time is 2022-05-11T15:17, date_first_hour_day (2022-05-11T00:00) will generate these views: + - standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep + - standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep + - standard_20220511 -> end date is 2022_05_12 T00:00, in future -> keep + - standard_2022051100 -> end date is 2022_05_01 T01:00, within TTL -> keep + */ + }, + { + name: "date_last_hour_day", + date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 23, 59), + expViews: []string{ + "standard", + "standard_" + fmt.Sprintf("%d", dateNow.Year()), + "standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()), + "standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()), + "standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 23), + }, + /* Example: current time is 2022-05-11T15:17, date_last_hour_day (2022-05-11T23:59) will generate these views: + - standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep + - standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep + - standard_20220511 -> end date is 2022_05_12 T00:00, in future -> keep + - standard_2022051123 -> end date is 2022_05_12 T00:00, in future-> keep + // */ + }, } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := client.CreateFieldWithOptions(context.Background(), indexName, test.name, pilosa.FieldOptions{TTL: time.Hour * 24, Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}); err != nil { + t.Fatalf("creating field, err: %v", err) + } - dateNow := time.Now() - dateNowString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour(), dateNow.Minute()) - /* Set sample data 2 using current time. - For example: current time is 2022-03-03T15:17 (also when the 24 hrs ttl countdown starts) will generate these views: - - standard_2022 -> gets converted to 2022_01_01, over 24 hours for ttl -> deleted - - standard_202203 -> gets converted to 2022_03_01, over 24 hours for ttl -> deleted - - standard_20220303 -> gets converted to 2022_03_03, within 24 hours -> keep - - standard_2022030315 -> gets converted to 2022_03_03 15:00, within 24 hours -> keep - */ - _, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(2, f=2, " + dateNowString + ")"}) - if err != nil { - t.Fatalf("setting sample data 2, err: %v", err) - } + // set data + _, err := client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(1, " + test.name + "=1, " + test.date + ")"}) + if err != nil { + t.Fatalf("setting sample data, err: %v", err) + } - /* Set sample data 3 using yesterday's date - All views generated from this date should be deleted - */ - dateYesterday := time.Now().Add(-24*time.Hour + -1*time.Nanosecond) - dateYesterdayString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour(), dateYesterday.Minute()) - _, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(3, f=3, " + dateYesterdayString + ")"}) - if err != nil { - t.Fatalf("setting sample data 3, err: %v", err) - } + // run TTLRemoval + node.Server.TTLRemoval(context.Background()) - node.Server.TTLRemoval(context.Background()) + // Get all the views for given index + field + views, err := node.API.Views(context.Background(), indexName, test.name) + if err != nil { + t.Fatal(err) + } + var viewNames []string + for _, view := range views { + viewNames = append(viewNames, view.Name()) + } + sort.Strings(viewNames) - // Get all the views for given index + field - views, err := node.API.Views(context.Background(), indexName, fieldName) - if err != nil { - t.Fatal(err) - } - - expectedViewNames := []string{ - "standard", - "standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()), - "standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour()), - } - - var viewNames []string - for _, view := range views { - viewNames = append(viewNames, view.Name()) - } - sort.Strings(viewNames) - - if !reflect.DeepEqual(expectedViewNames, viewNames) { - t.Fatalf("after ttl removal, expected %v, but got %v", expectedViewNames, viewNames) + if !reflect.DeepEqual(test.expViews, viewNames) { + t.Fatalf("after ttl removal, expected %v, but got %v", test.expViews, viewNames) + } + }) } }