mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
39006396db
* FB- 1456 - TTL - fixed views not returning correct results when least precise quantum are deleted * FB-1456 - TTL - PR - fixed comment * FB-1456 - TTL - changed getQuantum to getLowestGranularityQuantum since we only care about the least precise quantum that is available * FB-1456 - TTL - removed unit test used for debug * FB-1456 - TTL - fixed comments * [FB-1435] BSI Base Fix (#2056) * add bsi base back to int value * test bsi base/min/max for IntFields motivated by bsi base not being added back to values in extract calls when min was a positive integer. * FB-1456 - TTL - fixed comments Co-authored-by: Samir Patel <48686912+54mir@users.noreply.github.com>
188 lines
8.3 KiB
Go
188 lines
8.3 KiB
Go
// Copyright 2022 Molecula Corp. All rights reserved.
|
|
package pilosa_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"reflect"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
|
|
pilosa "github.com/molecula/featurebase/v3"
|
|
"github.com/molecula/featurebase/v3/test"
|
|
)
|
|
|
|
func TestTTLRemoval(t *testing.T) {
|
|
cluster := test.MustRunCluster(t, 1)
|
|
node := cluster.GetNode(0)
|
|
defer cluster.Close()
|
|
|
|
// Create a client
|
|
client := node.Client()
|
|
|
|
indexName := "i"
|
|
|
|
// Create indexes and field with ttl lasting 24 hours
|
|
if err := client.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{TrackExistence: true}); err != nil && err != pilosa.ErrIndexExists {
|
|
t.Fatalf("creating index, err: %v", err)
|
|
}
|
|
|
|
dateNow := time.Now().UTC()
|
|
dateYesterday := dateNow.Add(-24 * time.Hour)
|
|
dateCurrentMonth := time.Date(dateNow.Year(), dateNow.Month(), 1, 0, 0, 0, 0, time.UTC)
|
|
dateLastDayOfMonth := dateCurrentMonth.AddDate(0, 1, 0).Add(-time.Nanosecond)
|
|
|
|
var tests = []struct {
|
|
name string
|
|
date string
|
|
expViews []string
|
|
}{
|
|
{
|
|
name: "date_old",
|
|
date: "2001-02-03T04:05",
|
|
expViews: []string{"standard"},
|
|
/* date_old (2001-02-03T04:05), this will create these views:
|
|
- standard
|
|
- standard_2001
|
|
- standard_200102
|
|
- standard_20010203
|
|
- standard_2001020304
|
|
Since the sample date here is over 20 years all views except "standard" should get deleted
|
|
*/
|
|
},
|
|
{
|
|
name: "date_now",
|
|
date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour(), dateNow.Minute()),
|
|
expViews: []string{
|
|
"standard",
|
|
"standard_" + fmt.Sprintf("%d", dateNow.Year()),
|
|
"standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()),
|
|
"standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()),
|
|
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour()),
|
|
},
|
|
/* For example: current time is 2022-05-11T15:17 (also when the 24 hrs ttl countdown starts) will generate these views:
|
|
- standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
|
- standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
|
- standard_20220511 -> end date is 2022_05_12 T00:00, in future -> keep
|
|
- standard_2022051115 -> end date is 2022_05_11 T18:00, in future -> keep
|
|
*/
|
|
},
|
|
{
|
|
name: "date_yesterday",
|
|
date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour(), dateYesterday.Minute()),
|
|
expViews: []string{
|
|
"standard",
|
|
"standard_" + fmt.Sprintf("%d", dateYesterday.Year()),
|
|
"standard_" + fmt.Sprintf("%d%02d", dateYesterday.Year(), dateYesterday.Month()),
|
|
"standard_" + fmt.Sprintf("%d%02d%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day()),
|
|
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour()),
|
|
},
|
|
/* Example: current time is 2022-05-11T15:17, date_yesterday (2022-05-10T15:17) will generate these views:
|
|
- standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
|
- standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
|
- standard_20220510 -> end date is 2022_05_11 T00:00, within 24 hrs -> keep
|
|
- standard_2022051015 -> end date is 2022_05_10 T16:00, within 24 hrs -> keep
|
|
*/
|
|
},
|
|
// {
|
|
// name: "date_first_of_month",
|
|
// date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateCurrentMonth.Year(), dateCurrentMonth.Month(), 1, 0, 0),
|
|
// expViews: []string{
|
|
// "standard",
|
|
// "standard_" + fmt.Sprintf("%d", dateCurrentMonth.Year()),
|
|
// "standard_" + fmt.Sprintf("%d%02d", dateCurrentMonth.Year(), dateCurrentMonth.Month()),
|
|
// },
|
|
// /* Example: current time is 2022-05-11T15:17, date_first_of_month (2022-05-01T00:00) will generate these views:
|
|
// - standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
|
// - standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
|
// - standard_20220501 -> end date is 2022_05_02 T00:00, older than ttl -> delete
|
|
// - standard_2022050115 -> end date is 2022_05_01 T01:00, older than ttl -> delete
|
|
// !!! commenting this test out for now, there is a special case where if today's date is also same as date_first_of_month
|
|
// in that case, the expected views would have all 4, instead of 2 in the above example, since they are not older than the 24 hr ttl
|
|
// */
|
|
// },
|
|
{
|
|
name: "date_last_of_month",
|
|
date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateLastDayOfMonth.Year(), dateLastDayOfMonth.Month(), dateLastDayOfMonth.Day(), dateLastDayOfMonth.Hour(), dateLastDayOfMonth.Minute()),
|
|
expViews: []string{
|
|
"standard",
|
|
"standard_" + fmt.Sprintf("%d", dateLastDayOfMonth.Year()),
|
|
"standard_" + fmt.Sprintf("%d%02d", dateLastDayOfMonth.Year(), dateLastDayOfMonth.Month()),
|
|
"standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateLastDayOfMonth.Day()),
|
|
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateLastDayOfMonth.Day(), dateLastDayOfMonth.Hour()),
|
|
},
|
|
/* Example: current time is 2022-05-11T15:17, date_last_of_month (2022-05-31T23:59) will generate these views:
|
|
- standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
|
- standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
|
- standard_20220531 -> end date is 2022_06_01 T00:00, in future -> keep
|
|
- standard_2022053123 -> end date is 2022_06_01 T00:00, in future -> keep
|
|
*/
|
|
},
|
|
{
|
|
name: "date_first_hour_day",
|
|
date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 0, 0),
|
|
expViews: []string{
|
|
"standard",
|
|
"standard_" + fmt.Sprintf("%d", dateNow.Year()),
|
|
"standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()),
|
|
"standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()),
|
|
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 0),
|
|
},
|
|
/* Example: current time is 2022-05-11T15:17, date_first_hour_day (2022-05-11T00:00) will generate these views:
|
|
- standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
|
- standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
|
- standard_20220511 -> end date is 2022_05_12 T00:00, in future -> keep
|
|
- standard_2022051100 -> end date is 2022_05_01 T01:00, within TTL -> keep
|
|
*/
|
|
},
|
|
{
|
|
name: "date_last_hour_day",
|
|
date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 23, 59),
|
|
expViews: []string{
|
|
"standard",
|
|
"standard_" + fmt.Sprintf("%d", dateNow.Year()),
|
|
"standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()),
|
|
"standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()),
|
|
"standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 23),
|
|
},
|
|
/* Example: current time is 2022-05-11T15:17, date_last_hour_day (2022-05-11T23:59) will generate these views:
|
|
- standard_2022 -> end date is 2023_01_01 T00:00, in future -> keep
|
|
- standard_202205 -> end date is 2022_06_01 T00:00, in future -> keep
|
|
- standard_20220511 -> end date is 2022_05_12 T00:00, in future -> keep
|
|
- standard_2022051123 -> end date is 2022_05_12 T00:00, in future -> keep
|
|
*/
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
if err := client.CreateFieldWithOptions(context.Background(), indexName, test.name, pilosa.FieldOptions{TTL: time.Hour * 24, Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}); err != nil {
|
|
t.Fatalf("creating field, err: %v", err)
|
|
}
|
|
|
|
// set data
|
|
_, err := client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(1, " + test.name + "=1, " + test.date + ")"})
|
|
if err != nil {
|
|
t.Fatalf("setting sample data, err: %v", err)
|
|
}
|
|
|
|
// run TTLRemoval
|
|
node.Server.TTLRemoval(context.Background())
|
|
|
|
// Get all the views for given index + field
|
|
views, err := node.API.Views(context.Background(), indexName, test.name)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var viewNames []string
|
|
for _, view := range views {
|
|
viewNames = append(viewNames, view.Name())
|
|
}
|
|
sort.Strings(viewNames)
|
|
|
|
if !reflect.DeepEqual(test.expViews, viewNames) {
|
|
t.Fatalf("after ttl removal, expected %v, but got %v", test.expViews, viewNames)
|
|
}
|
|
})
|
|
}
|
|
}
|