diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index cb38aee26..8c165da1c 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -615,19 +615,20 @@ func (s Serializer) encodeFieldOptions(o *pilosa.FieldOptions) *pb.FieldOptions return nil } return &pb.FieldOptions{ - Type: o.Type, - CacheType: o.CacheType, - CacheSize: o.CacheSize, - Min: s.encodeDecimal(&o.Min), - Max: s.encodeDecimal(&o.Max), - Base: o.Base, - Scale: o.Scale, - BitDepth: uint64(o.BitDepth), - TimeQuantum: string(o.TimeQuantum), - TTL: o.TTL.String(), - TimeUnit: string(o.TimeUnit), - Keys: o.Keys, - ForeignIndex: o.ForeignIndex, + Type: o.Type, + CacheType: o.CacheType, + CacheSize: o.CacheSize, + Min: s.encodeDecimal(&o.Min), + Max: s.encodeDecimal(&o.Max), + Base: o.Base, + Scale: o.Scale, + BitDepth: uint64(o.BitDepth), + TimeQuantum: string(o.TimeQuantum), + TTL: o.TTL.String(), + TimeUnit: string(o.TimeUnit), + Keys: o.Keys, + ForeignIndex: o.ForeignIndex, + NoStandardView: o.NoStandardView, } } @@ -996,6 +997,7 @@ func (s Serializer) decodeFieldOptions(options *pb.FieldOptions, m *pilosa.Field m.TimeUnit = options.TimeUnit m.Keys = options.Keys m.ForeignIndex = options.ForeignIndex + m.NoStandardView = options.NoStandardView } func (s Serializer) decodeDecimal(d *pb.Decimal, m *pql.Decimal) { diff --git a/http_handler_test.go b/http_handler_test.go index 37aca40b2..a58d3742b 100644 --- a/http_handler_test.go +++ b/http_handler_test.go @@ -86,7 +86,7 @@ func TestMarshalUnmarshalTransactionResponse(t *testing.T) { } } -func TestUpdateFieldTTL(t *testing.T) { +func TestUpdateField_TTL(t *testing.T) { c := test.MustRunCluster(t, 3) defer c.Close() @@ -226,6 +226,106 @@ func TestUpdateFieldTTL(t *testing.T) { } +func TestUpdateField_NoStandardView(t *testing.T) { + c := test.MustRunCluster(t, 3) + defer c.Close() + + tests := []struct { + name string + field string + fieldOption string + expStatus int + expErr string + expNoStandardView bool + }{ + { + // test update noStandardView to true + name: "t1_true", + field: "t1_true", + fieldOption: `{"option": "noStandardView", "value": "true"}`, + expStatus: 200, + expErr: "", + expNoStandardView: true, + }, + { + // test update noStandardView to false + name: "t2_false", + field: "t2_false", + fieldOption: `{"option": "noStandardView", "value": "false"}`, + expStatus: 200, + expErr: "", + expNoStandardView: false, + }, + { + // test update noStandardView with invalid value + name: "t3_invalid", + field: "t3_invalid", + fieldOption: `{"option": "noStandardView", "value": "123"}`, + expStatus: 400, + expErr: `invalid value for noStandardView: '123'`, + expNoStandardView: false, + }, + { + // test udpate noStandardView with empty value + name: "t4_empty", + field: "t4_empty", + fieldOption: `{"option": "noStandardView", "value": ""}`, + expStatus: 400, + expErr: `invalid value for noStandardView: ''`, + expNoStandardView: false, + }, + } + + for i, test := range tests { + t.Run(test.name, func(t *testing.T) { + c.CreateField(t, "ttltest", pilosa.IndexOptions{}, test.name, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0")) + nodeURL := c.Nodes[0].URL() + "/index/ttltest/field/" + test.field + req, err := gohttp.NewRequest("PATCH", nodeURL, strings.NewReader(test.fieldOption)) + if err != nil { + t.Fatal(err) + } + + req.Header.Set("Content-Type", "application/json") + resp, err := gohttp.DefaultClient.Do(req) + + if err != nil { + t.Fatalf("doing option request: %v", err) + } + + if resp.StatusCode != test.expStatus { + t.Errorf("expected status: '%d', got: '%d'", test.expStatus, resp.StatusCode) + } + + if resp.StatusCode == 400 { + // unmarshal error message to check against expErr + var respBody map[string]interface{} + json.NewDecoder(resp.Body).Decode(&respBody) + errMsg := respBody["error"].(map[string]interface{})["message"].(string) + + if !strings.Contains(errMsg, test.expErr) { + t.Errorf("expected error: '%s', got: '%s'", test.expErr, errMsg) + } + } + + // find updated field and check its noStandardView + for _, node := range c.Nodes { + ii, err := node.API.Schema(context.Background(), false) + if err != nil { + t.Fatalf("getting schema: %v", err) + } + if ii[0].Fields[i].Name == test.name { + if ii[0].Fields[i].Options.NoStandardView != test.expNoStandardView { + t.Errorf("expected noStandardView value: '%t', got: '%t'", test.expNoStandardView, ii[0].Fields[i].Options.NoStandardView) + } + } else { + t.Errorf("unexpected field: '%s', got: '%s'", test.name, ii[0].Fields[i].Name) + } + } + + }) + } +} + func TestIngestSchemaHandler(t *testing.T) { c := test.MustRunCluster(t, 3, []server.CommandOption{ diff --git a/index.go b/index.go index 7c05e8c66..6e4d6c1a9 100644 --- a/index.go +++ b/index.go @@ -772,6 +772,15 @@ func (i *Index) UpdateField(ctx context.Context, name string, update FieldUpdate return nil, NewBadRequestError(errors.Errorf("ttl can't be negative: '%s'", update.Value)) } cfm.Meta.TTL = dur + case "noStandardView": + if cfm.Meta.Type != FieldTypeTime { + return nil, NewBadRequestError(errors.Errorf("can only update 'noStandardView' on a 'time' type field, not '%s'", cfm.Meta.Type)) + } + boolValue, err := strconv.ParseBool(update.Value) + if err != nil { + return nil, NewBadRequestError(errors.Errorf("invalid value for noStandardView: '%s'", update.Value)) + } + cfm.Meta.NoStandardView = boolValue default: return nil, NewBadRequestError(errors.Errorf("updates for option '%s' are not supported", update.Option)) } diff --git a/server.go b/server.go index e54788b4d..c83455df3 100644 --- a/server.go +++ b/server.go @@ -895,6 +895,21 @@ func (s *Server) TTLRemoval(ctx context.Context) { } } } + if field.Options().NoStandardView { + // delete view "standard" if NoStandardView is true + for _, shard := range field.AvailableShards(true).Slice() { + err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), viewStandard, shard, nil) + if err != nil { + s.logger.Errorf("delete view %s from shard %d: %s", viewStandard, shard, err) + } + } + + err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewStandard) + if err != nil { + s.logger.Errorf("view: %s, delete view: %s", viewStandard, err) + } + s.logger.Infof("view %s deleted - index: %s, field: %s ", viewStandard, index.name, field.name) + } } } } diff --git a/server_test.go b/server_test.go index 3b92f12ea..528bc5adb 100644 --- a/server_test.go +++ b/server_test.go @@ -13,7 +13,7 @@ import ( "github.com/molecula/featurebase/v3/test" ) -func TestTTLRemoval(t *testing.T) { +func TestTTLRemoval_TTL(t *testing.T) { cluster := test.MustRunCluster(t, 1) node := cluster.GetNode(0) defer cluster.Close() @@ -186,3 +186,94 @@ func TestTTLRemoval(t *testing.T) { }) } } + +func TestTTLRemoval_StandardView(t *testing.T) { + cluster := test.MustRunCluster(t, 1) + node := cluster.GetNode(0) + defer cluster.Close() + + // Create a client + client := node.Client() + + indexName := "i" + + // Create indexes and field with ttl lasting 24 hours + if err := client.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{TrackExistence: true}); err != nil && err != pilosa.ErrIndexExists { + t.Fatalf("creating index, err: %v", err) + } + + var tests = []struct { + name string + date string + noStandardView string + expViews []string + }{ + { + name: "t1_delete_standard", + date: "2001-02-03T04:05", + noStandardView: "true", + expViews: nil, + /* date 2001-02-03T04:05, this will create these views: + - standard + - standard_2001 + - standard_200102 + - standard_20010203 + - standard_2001020304 + Since the sample date here is over 20 years, all views with dates should get deleted + Since noStandardView is true, 'standard' view should get deleted + */ + }, + { + name: "t2_keep_standard", + date: "2001-02-03T04:05", + noStandardView: "false", + expViews: []string{"standard"}, + /* date 2001-02-03T04:05, this will create these views: + - standard + - standard_2001 + - standard_200102 + - standard_20010203 + - standard_2001020304 + Since the sample date here is over 20 years, all views with dates should get deleted + Since noStandardView is false, 'standard' view should NOT get deleted + */ + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := client.CreateFieldWithOptions(context.Background(), indexName, test.name, pilosa.FieldOptions{TTL: time.Hour * 24, Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}); err != nil { + t.Fatalf("creating field, err: %v", err) + } + + // set data + _, err := client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(1, " + test.name + "=1, " + test.date + ")"}) + if err != nil { + t.Fatalf("setting sample data, err: %v", err) + } + + // update noStandardView value + err = node.API.UpdateField(context.Background(), indexName, test.name, pilosa.FieldUpdate{Option: "noStandardView", Value: test.noStandardView}) + if err != nil { + t.Fatalf("updating noStandardView, err: %v", err) + } + + // run TTLRemoval + node.Server.TTLRemoval(context.Background()) + + // get all the views for given index + field + views, err := node.API.Views(context.Background(), indexName, test.name) + if err != nil { + t.Fatal(err) + } + var viewNames []string + for _, view := range views { + viewNames = append(viewNames, view.Name()) + } + sort.Strings(viewNames) + if !reflect.DeepEqual(test.expViews, viewNames) { + t.Fatalf("after ttl removal, expected %v, but got %v", test.expViews, viewNames) + } + }) + } +}