FB-1505 - added ability to update NoStandardView and delete standard view

encoding/proto/proto.go

@@ -615,19 +615,20 @@ func (s Serializer) encodeFieldOptions(o *pilosa.FieldOptions) *pb.FieldOptions
 		return nil
 	}
 	return &pb.FieldOptions{
-		Type:         o.Type,
-		CacheType:    o.CacheType,
-		CacheSize:    o.CacheSize,
-		Min:          s.encodeDecimal(&o.Min),
-		Max:          s.encodeDecimal(&o.Max),
-		Base:         o.Base,
-		Scale:        o.Scale,
-		BitDepth:     uint64(o.BitDepth),
-		TimeQuantum:  string(o.TimeQuantum),
-		TTL:          o.TTL.String(),
-		TimeUnit:     string(o.TimeUnit),
-		Keys:         o.Keys,
-		ForeignIndex: o.ForeignIndex,
+		Type:           o.Type,
+		CacheType:      o.CacheType,
+		CacheSize:      o.CacheSize,
+		Min:            s.encodeDecimal(&o.Min),
+		Max:            s.encodeDecimal(&o.Max),
+		Base:           o.Base,
+		Scale:          o.Scale,
+		BitDepth:       uint64(o.BitDepth),
+		TimeQuantum:    string(o.TimeQuantum),
+		TTL:            o.TTL.String(),
+		TimeUnit:       string(o.TimeUnit),
+		Keys:           o.Keys,
+		ForeignIndex:   o.ForeignIndex,
+		NoStandardView: o.NoStandardView,
 	}
 }
 
@@ -996,6 +997,7 @@ func (s Serializer) decodeFieldOptions(options *pb.FieldOptions, m *pilosa.Field
 	m.TimeUnit = options.TimeUnit
 	m.Keys = options.Keys
 	m.ForeignIndex = options.ForeignIndex
+	m.NoStandardView = options.NoStandardView
 }
 
 func (s Serializer) decodeDecimal(d *pb.Decimal, m *pql.Decimal) {

http_handler_test.go

@@ -86,7 +86,7 @@ func TestMarshalUnmarshalTransactionResponse(t *testing.T) {
 	}
 }
 
-func TestUpdateFieldTTL(t *testing.T) {
+func TestUpdateField_TTL(t *testing.T) {
 	c := test.MustRunCluster(t, 3)
 	defer c.Close()
 
@@ -226,6 +226,106 @@ func TestUpdateFieldTTL(t *testing.T) {
 
 }
 
+func TestUpdateField_NoStandardView(t *testing.T) {
+	c := test.MustRunCluster(t, 3)
+	defer c.Close()
+
+	tests := []struct {
+		name              string
+		field             string
+		fieldOption       string
+		expStatus         int
+		expErr            string
+		expNoStandardView bool
+	}{
+		{
+			// test update noStandardView to true
+			name:              "t1_true",
+			field:             "t1_true",
+			fieldOption:       `{"option": "noStandardView", "value": "true"}`,
+			expStatus:         200,
+			expErr:            "",
+			expNoStandardView: true,
+		},
+		{
+			// test update noStandardView to false
+			name:              "t2_false",
+			field:             "t2_false",
+			fieldOption:       `{"option": "noStandardView", "value": "false"}`,
+			expStatus:         200,
+			expErr:            "",
+			expNoStandardView: false,
+		},
+		{
+			// test update noStandardView with invalid value
+			name:              "t3_invalid",
+			field:             "t3_invalid",
+			fieldOption:       `{"option": "noStandardView", "value": "123"}`,
+			expStatus:         400,
+			expErr:            `invalid value for noStandardView: '123'`,
+			expNoStandardView: false,
+		},
+		{
+			// test udpate noStandardView with empty value
+			name:              "t4_empty",
+			field:             "t4_empty",
+			fieldOption:       `{"option": "noStandardView", "value": ""}`,
+			expStatus:         400,
+			expErr:            `invalid value for noStandardView: ''`,
+			expNoStandardView: false,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			c.CreateField(t, "ttltest", pilosa.IndexOptions{}, test.name, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0"))
+			nodeURL := c.Nodes[0].URL() + "/index/ttltest/field/" + test.field
+			req, err := gohttp.NewRequest("PATCH", nodeURL, strings.NewReader(test.fieldOption))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			req.Header.Set("Content-Type", "application/json")
+			resp, err := gohttp.DefaultClient.Do(req)
+
+			if err != nil {
+				t.Fatalf("doing option request: %v", err)
+			}
+
+			if resp.StatusCode != test.expStatus {
+				t.Errorf("expected status: '%d', got: '%d'", test.expStatus, resp.StatusCode)
+			}
+
+			if resp.StatusCode == 400 {
+				// unmarshal error message to check against expErr
+				var respBody map[string]interface{}
+				json.NewDecoder(resp.Body).Decode(&respBody)
+				errMsg := respBody["error"].(map[string]interface{})["message"].(string)
+
+				if !strings.Contains(errMsg, test.expErr) {
+					t.Errorf("expected error: '%s', got: '%s'", test.expErr, errMsg)
+				}
+			}
+
+			// find updated field and check its noStandardView
+			for _, node := range c.Nodes {
+				ii, err := node.API.Schema(context.Background(), false)
+				if err != nil {
+					t.Fatalf("getting schema: %v", err)
+				}
+				if ii[0].Fields[i].Name == test.name {
+					if ii[0].Fields[i].Options.NoStandardView != test.expNoStandardView {
+						t.Errorf("expected noStandardView value: '%t', got: '%t'", test.expNoStandardView, ii[0].Fields[i].Options.NoStandardView)
+					}
+				} else {
+					t.Errorf("unexpected field: '%s', got: '%s'", test.name, ii[0].Fields[i].Name)
+				}
+			}
+
+		})
+	}
+}
+
 func TestIngestSchemaHandler(t *testing.T) {
 	c := test.MustRunCluster(t, 3,
 		[]server.CommandOption{

index.go

@@ -772,6 +772,15 @@ func (i *Index) UpdateField(ctx context.Context, name string, update FieldUpdate
 			return nil, NewBadRequestError(errors.Errorf("ttl can't be negative: '%s'", update.Value))
 		}
 		cfm.Meta.TTL = dur
+	case "noStandardView":
+		if cfm.Meta.Type != FieldTypeTime {
+			return nil, NewBadRequestError(errors.Errorf("can only update 'noStandardView' on a 'time' type field, not '%s'", cfm.Meta.Type))
+		}
+		boolValue, err := strconv.ParseBool(update.Value)
+		if err != nil {
+			return nil, NewBadRequestError(errors.Errorf("invalid value for noStandardView: '%s'", update.Value))
+		}
+		cfm.Meta.NoStandardView = boolValue
 	default:
 		return nil, NewBadRequestError(errors.Errorf("updates for option '%s' are not supported", update.Option))
 	}

server.go

@@ -895,6 +895,21 @@ func (s *Server) TTLRemoval(ctx context.Context) {
 						}
 					}
 				}
+				if field.Options().NoStandardView {
+					// delete view "standard" if NoStandardView is true
+					for _, shard := range field.AvailableShards(true).Slice() {
+						err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), viewStandard, shard, nil)
+						if err != nil {
+							s.logger.Errorf("delete view %s from shard %d: %s", viewStandard, shard, err)
+						}
+					}
+
+					err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewStandard)
+					if err != nil {
+						s.logger.Errorf("view: %s, delete view: %s", viewStandard, err)
+					}
+					s.logger.Infof("view %s deleted - index: %s, field: %s ", viewStandard, index.name, field.name)
+				}
 			}
 		}
 	}

server_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/molecula/featurebase/v3/test"
 )
 
-func TestTTLRemoval(t *testing.T) {
+func TestTTLRemoval_TTL(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	node := cluster.GetNode(0)
 	defer cluster.Close()
@@ -186,3 +186,94 @@ func TestTTLRemoval(t *testing.T) {
 		})
 	}
 }
+
+func TestTTLRemoval_StandardView(t *testing.T) {
+	cluster := test.MustRunCluster(t, 1)
+	node := cluster.GetNode(0)
+	defer cluster.Close()
+
+	// Create a client
+	client := node.Client()
+
+	indexName := "i"
+
+	// Create indexes and field with ttl lasting 24 hours
+	if err := client.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{TrackExistence: true}); err != nil && err != pilosa.ErrIndexExists {
+		t.Fatalf("creating index, err: %v", err)
+	}
+
+	var tests = []struct {
+		name           string
+		date           string
+		noStandardView string
+		expViews       []string
+	}{
+		{
+			name:           "t1_delete_standard",
+			date:           "2001-02-03T04:05",
+			noStandardView: "true",
+			expViews:       nil,
+			/* date 2001-02-03T04:05, this will create these views:
+			- standard
+			- standard_2001
+			- standard_200102
+			- standard_20010203
+			- standard_2001020304
+			Since the sample date here is over 20 years, all views with dates should get deleted
+			Since noStandardView is true, 'standard' view should get deleted
+			*/
+		},
+		{
+			name:           "t2_keep_standard",
+			date:           "2001-02-03T04:05",
+			noStandardView: "false",
+			expViews:       []string{"standard"},
+			/* date 2001-02-03T04:05, this will create these views:
+			- standard
+			- standard_2001
+			- standard_200102
+			- standard_20010203
+			- standard_2001020304
+			Since the sample date here is over 20 years, all views with dates should get deleted
+			Since noStandardView is false, 'standard' view should NOT get deleted
+			*/
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if err := client.CreateFieldWithOptions(context.Background(), indexName, test.name, pilosa.FieldOptions{TTL: time.Hour * 24, Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}); err != nil {
+				t.Fatalf("creating field, err: %v", err)
+			}
+
+			// set data
+			_, err := client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(1, " + test.name + "=1, " + test.date + ")"})
+			if err != nil {
+				t.Fatalf("setting sample data, err: %v", err)
+			}
+
+			// update noStandardView value
+			err = node.API.UpdateField(context.Background(), indexName, test.name, pilosa.FieldUpdate{Option: "noStandardView", Value: test.noStandardView})
+			if err != nil {
+				t.Fatalf("updating noStandardView, err: %v", err)
+			}
+
+			// run TTLRemoval
+			node.Server.TTLRemoval(context.Background())
+
+			// get all the views for given index + field
+			views, err := node.API.Views(context.Background(), indexName, test.name)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var viewNames []string
+			for _, view := range views {
+				viewNames = append(viewNames, view.Name())
+			}
+			sort.Strings(viewNames)
+			if !reflect.DeepEqual(test.expViews, viewNames) {
+				t.Fatalf("after ttl removal, expected %v, but got %v", test.expViews, viewNames)
+			}
+		})
+	}
+}