From 6d18d9e0c42518742610dab30a6faf5a7dccc0cf Mon Sep 17 00:00:00 2001 From: Hoang Pham Date: Thu, 3 Mar 2022 16:09:37 -0600 Subject: [PATCH] FB-1188 - added more unit tests --- server.go | 67 ++++++++++++++++++++--------------- server_test.go | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++ time.go | 2 +- view.go | 4 +++ 4 files changed, 139 insertions(+), 29 deletions(-) create mode 100644 server_test.go diff --git a/server.go b/server.go index 1238135c6..32c7a0e1f 100644 --- a/server.go +++ b/server.go @@ -75,6 +75,7 @@ type Server struct { // nolint: maligned antiEntropyInterval time.Duration metricInterval time.Duration diagnosticInterval time.Duration + ttlRemovalInterval time.Duration maxWritesPerRequest int confirmDownSleep time.Duration confirmDownRetries int @@ -146,6 +147,15 @@ func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { } } +// OptServerTtlRemovalInterval is a functional option on Server +// used to set the ttl removal interval. +func OptServerTtlRemovalInterval(interval time.Duration) ServerOption { + return func(s *Server) error { + s.ttlRemovalInterval = interval + return nil + } +} + // OptServerLongQueryTime is a functional option on Server // used to set long query duration. func OptServerLongQueryTime(dur time.Duration) ServerOption { @@ -412,6 +422,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { antiEntropyInterval: 0, metricInterval: 0, diagnosticInterval: 0, + ttlRemovalInterval: time.Hour, disCo: disco.NopDisCo, stator: disco.NopStator, @@ -796,42 +807,42 @@ func (s *Server) monitorResetTranslationSync() { func (s *Server) monitorTtl() { ctx := context.Background() - ttlRemovalInterval, err := time.ParseDuration("1h") - if err != nil { - s.logger.Errorf("ttl parse interval error: err=%s", err) - } - ticker := time.NewTicker(ttlRemovalInterval) + ticker := time.NewTicker(s.ttlRemovalInterval) for { select { case <-s.closing: return case <-ticker.C: - for _, index := range s.holder.Indexes() { - for _, field := range index.Fields() { - if field.Options().Type == "time" { - if field.Options().Ttl > 0 { - for _, view := range field.views() { - viewNames := strings.Split(view.name, "_") - if len(viewNames) >= 2 { - viewTime, err := timeOfView(view.name, false) - if err != nil { - s.logger.Printf("ttl parse view time: %s", err) - continue - } - timeSince := time.Since(viewTime) + s.TtlRemoval(ctx) + } + } +} - if timeSince > field.Options().Ttl { - for _, shard := range field.AvailableShards(true).Slice() { - s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), view.name, shard, nil) - } +func (s *Server) TtlRemoval(ctx context.Context) { + for _, index := range s.holder.Indexes() { + for _, field := range index.Fields() { + if field.Options().Type == "time" { + if field.Options().Ttl > 0 { + for _, view := range field.views() { + viewNames := strings.Split(view.name, "_") + if len(viewNames) >= 2 { + viewTime, err := timeOfView(view.name, false) + if err != nil { + s.logger.Printf("ttl parse view time: %s", err) + continue + } + timeSince := time.Since(viewTime) - err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), view.name) - if err != nil { - s.logger.Errorf("ttl delete view: %s", err) - } - s.logger.Infof("ttl deleted view: %s", view.name) - } + if timeSince >= field.Options().Ttl { + for _, shard := range field.AvailableShards(true).Slice() { + s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), view.name, shard, nil) } + + err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), view.name) + if err != nil { + s.logger.Errorf("ttl delete view: %s", err) + } + s.logger.Infof("ttl deleted view: %s", view.name) } } } diff --git a/server_test.go b/server_test.go new file mode 100644 index 000000000..487a39286 --- /dev/null +++ b/server_test.go @@ -0,0 +1,95 @@ +// Copyright 2022 Molecula Corp. All rights reserved. +package pilosa_test + +import ( + "context" + "fmt" + "reflect" + "sort" + "testing" + "time" + + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/test" +) + +func TestTtlRemoval(t *testing.T) { + + cluster := test.MustRunCluster(t, 1) + node := cluster.GetNode(0) + defer cluster.Close() + + // Create a client + client := node.Client() + + indexName := "i" + fieldName := "f" + + // Create indexes and field with ttl lasting 24 hours + if err := client.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{TrackExistence: true}); err != nil && err != pilosa.ErrIndexExists { + t.Fatalf("creating index, err: %v", err) + } else if err := client.CreateFieldWithOptions(context.Background(), indexName, fieldName, pilosa.FieldOptions{Ttl: time.Hour * 24, Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}); err != nil { + t.Fatalf("creating field, err: %v", err) + } + + /* Set sample data 1 using this date: '2001-02-03T04:05', this will create these views: + - standard + - standard_2001 + - standard_200102 + - standard_20010203 + - standard_2001020304 + Since the sample date here is over 24 hours, all views except "standard" should get deleted + */ + _, err := client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(1, f=1, 2001-02-03T04:05)"}) + if err != nil { + t.Fatalf("setting sample data 1, err: %v", err) + } + + dateNow := time.Now() + dateNowString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour(), dateNow.Minute()) + /* Set sample data 2 using current time. + For example: current time is 2022-03-03T15:17 (also when the 24 hrs ttl countdown starts) will generate these views: + - standard_2022 -> gets converted to 2022_01_01, over 24 hours for ttl -> deleted + - standard_202203 -> gets converted to 2022_03_01, over 24 hours for ttl -> deleted + - standard_20220303 -> gets converted to 2022_03_03, within 24 hours -> keep + - standard_2022030315 -> gets converted to 2022_03_03 15:00, within 24 hours -> keep + */ + _, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(2, f=2, " + dateNowString + ")"}) + if err != nil { + t.Fatalf("setting sample data 2, err: %v", err) + } + + /* Set sample data 3 using yesterday's date + All views generated from this date should be deleted + */ + dateYesterday := time.Now().Add(-24*time.Hour + -1*time.Nanosecond) + dateYesterdayString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour(), dateYesterday.Minute()) + _, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(3, f=3, " + dateYesterdayString + ")"}) + if err != nil { + t.Fatalf("setting sample data 3, err: %v", err) + } + + node.Server.TtlRemoval(context.Background()) + + // Get all the views for given index + field + views, err := node.API.Views(context.Background(), indexName, fieldName) + if err != nil { + t.Fatal(err) + } + + expectedViewNames := []string{ + "standard", + "standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()), + "standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour()), + } + + var viewNames []string + for _, view := range views { + viewNames = append(viewNames, view.Name()) + } + sort.Strings(viewNames) + + if !reflect.DeepEqual(expectedViewNames, viewNames) { + t.Fatalf("after ttl removal, expected %v, but got %v", expectedViewNames, viewNames) + } +} diff --git a/time.go b/time.go index 1c479c996..945d87106 100644 --- a/time.go +++ b/time.go @@ -450,7 +450,7 @@ func timeOfView(v string, adj bool) (time.Time, error) { return time.Time{}, nil } - layout := "2006010203" + layout := "2006010215" timePart := viewTimePart(v) switch len(timePart) { diff --git a/view.go b/view.go index d5e408810..927490a9f 100644 --- a/view.go +++ b/view.go @@ -307,6 +307,10 @@ func (v *view) recalculateCaches() { } } +func (v *view) Name() string { + return v.name +} + // CreateFragmentIfNotExists returns a fragment in the view by shard. func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) { v.mu.Lock()