From aff3d3ddd9248dc75f7ca121eed8b9ad83bcbb1f Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 7 Dec 2021 14:58:35 -0600 Subject: [PATCH] do a backup in a go test for coverage purposes also found a weird issue with schema marshalling if you create a field thru the api w/o specifying a field type, you get slightly different behavior than going thru the HTTP handler which is... not ideal. I changed the marshaler to accept an empty field type. --- ctl/backup.go | 2 +- executor_test.go | 73 ++++++++++++++++++++++++++++++++++++------------ field.go | 4 +-- 3 files changed, 58 insertions(+), 21 deletions(-) diff --git a/ctl/backup.go b/ctl/backup.go index 98d6140c9..4c6c006ec 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -18,7 +18,7 @@ import ( "golang.org/x/sync/errgroup" ) -// BackupCommand represents a command for backing up a Pilosa node. +// BackupCommand represents a command for backing up a FeatureBase node. type BackupCommand struct { // nolint: maligned tlsConfig *tls.Config diff --git a/executor_test.go b/executor_test.go index 56023107e..9c8aa595f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -27,6 +27,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" + "github.com/molecula/featurebase/v2/ctl" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/pql" @@ -7015,11 +7016,16 @@ func TestMissingKeyRegression(t *testing.T) { // (single and multi-node clusters, different endpoints for the // queries (HTTP, GRPC, Postgres), etc.). func TestVariousQueries(t *testing.T) { - for _, clusterSize := range []int{1, 3, 4, 7} { + for _, clusterSize := range []int{1, 3, 7} { clusterSize := clusterSize t.Run(fmt.Sprintf("%d-node", clusterSize), func(t *testing.T) { c := test.MustRunCluster(t, clusterSize) defer c.Close() + + // put a variety of data into the cluster + populateTestData(t, c) + backupTest(t, c) + variousQueries(t, c) variousQueriesOnTimeFields(t, c) variousQueriesOnPercentiles(t, c) @@ -7028,6 +7034,30 @@ func TestVariousQueries(t *testing.T) { } } +func backupTest(t *testing.T, c *test.Cluster) { + // should this really be in executor? No. But all these + // integration-y query tests probably shouldn't be either. My goal + // putting this here is to take advantage of already-existing + // clusters and data. + + td, err := testhook.TempDir(t, "backupTest") + if err != nil { + t.Fatalf("can't even get a temp dir, what a ripoff: %v", err) + } + td = td + "/backupTest" + + buf := &bytes.Buffer{} + backupCommand := ctl.NewBackupCommand(nil, buf, buf) + backupCommand.Host = c.Nodes[len(c.Nodes)-1].URL() // don't pick node 0 so we don't always get primary (better code coverage) + backupCommand.Index = usersIndex + backupCommand.OutputDir = td + + if err := backupCommand.Run(context.Background()); err != nil { + t.Log(buf.String()) + t.Fatalf("running backup: %v", err) + } +} + // tests for abbreviating time values in queries func variousQueriesOnPercentiles(t *testing.T, c *test.Cluster) { // todo, make rand more random, 42 isnt the answer to everything @@ -7332,10 +7362,12 @@ func variousQueriesOnTimeFields(t *testing.T, c *test.Cluster) { } } -func variousQueries(t *testing.T, c *test.Cluster) { +var usersIndex = "users" + +func populateTestData(t *testing.T, c *test.Cluster) { // Create and populate "likenums" similar to "likes", but without keys on the field. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums") - c.ImportIDKey(t, "users", "likenums", []test.KeyID{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums") + c.ImportIDKey(t, usersIndex, "likenums", []test.KeyID{ {ID: 1, Key: "userA"}, {ID: 2, Key: "userB"}, {ID: 3, Key: "userC"}, @@ -7353,8 +7385,8 @@ func variousQueries(t *testing.T, c *test.Cluster) { }) // Create and populate "likes" field. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys()) - c.ImportKeyKey(t, "users", "likes", [][2]string{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys()) + c.ImportKeyKey(t, usersIndex, "likes", [][2]string{ {"molecula", "userA"}, {"pilosa", "userB"}, {"pangolin", "userC"}, @@ -7370,8 +7402,8 @@ func variousQueries(t *testing.T, c *test.Cluster) { }) // Create and populate "dinner" field. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "dinner", pilosa.OptFieldKeys()) - c.ImportKeyKey(t, "users", "dinner", [][2]string{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "dinner", pilosa.OptFieldKeys()) + c.ImportKeyKey(t, usersIndex, "dinner", [][2]string{ {"leftovers", "userB"}, {"pizza", "userA"}, {"pizza", "userB"}, @@ -7381,11 +7413,11 @@ func variousQueries(t *testing.T, c *test.Cluster) { }) // Create and populate "places_visited" time field. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM"))) + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM"))) ts2019Jan01 := int64(1546300800) * 1e+9 // 2019 January 1st 0:00:00 ts2019Aug01 := int64(1564617600) * 1e+9 // 2019 August 1st 0:00:00 ts2020Jan01 := int64(1577836800) * 1e+9 // 2020 January 1st 0:00:00 - c.ImportTimeQuantumKey(t, "users", "places_visited", []test.TimeQuantumKey{ + c.ImportTimeQuantumKey(t, usersIndex, "places_visited", []test.TimeQuantumKey{ // 2019 January: nairobi, paris, austin, toronto {RowKey: "nairobi", ColKey: "userB", Ts: ts2019Jan01}, {RowKey: "paris", ColKey: "userC", Ts: ts2019Jan01}, @@ -7405,8 +7437,8 @@ func variousQueries(t *testing.T, c *test.Cluster) { }) // Create and populate "affinity" int field with negative, positive, zero and null values. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "affinity", pilosa.OptFieldTypeInt(-1000, 1000)) - c.ImportIntKey(t, "users", "affinity", []test.IntKey{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "affinity", pilosa.OptFieldTypeInt(-1000, 1000)) + c.ImportIntKey(t, usersIndex, "affinity", []test.IntKey{ {Val: 10, Key: "userA"}, {Val: -10, Key: "userB"}, {Val: 5, Key: "userC"}, @@ -7415,8 +7447,8 @@ func variousQueries(t *testing.T, c *test.Cluster) { }) // Create and populate "net_worth" int field with positive values. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "net_worth", pilosa.OptFieldTypeInt(-100000000, 100000000)) - c.ImportIntKey(t, "users", "net_worth", []test.IntKey{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "net_worth", pilosa.OptFieldTypeInt(-100000000, 100000000)) + c.ImportIntKey(t, usersIndex, "net_worth", []test.IntKey{ {Val: 1, Key: "userA"}, {Val: 10, Key: "userB"}, {Val: 100, Key: "userC"}, @@ -7425,8 +7457,8 @@ func variousQueries(t *testing.T, c *test.Cluster) { {Val: 100000, Key: "userF"}, }) - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "zip_code", pilosa.OptFieldTypeInt(0, 100000)) - c.ImportIntKey(t, "users", "zip_code", []test.IntKey{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "zip_code", pilosa.OptFieldTypeInt(0, 100000)) + c.ImportIntKey(t, usersIndex, "zip_code", []test.IntKey{ {Val: 78739, Key: "userA"}, {Val: 78739, Key: "userB"}, {Val: 19707, Key: "userC"}, @@ -7434,7 +7466,12 @@ func variousQueries(t *testing.T, c *test.Cluster) { {Val: 86753, Key: "userE"}, {Val: 78739, Key: "userG"}, }) +} +func variousQueries(t *testing.T, c *test.Cluster) { + // NOTE: this relies on populateTestData being called first + + // define and run a bunch of tests tests := []struct { query string qrVerifier func(t *testing.T, resp pilosa.QueryResponse) @@ -7781,8 +7818,8 @@ leftovers,1 for i, tst := range tests { t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) { - resp := c.Query(t, "users", tst.query) - tr := c.QueryGRPC(t, "users", tst.query) + resp := c.Query(t, usersIndex, tst.query) + tr := c.QueryGRPC(t, usersIndex, tst.query) if tst.qrVerifier != nil { tst.qrVerifier(t, resp) } diff --git a/field.go b/field.go index 7076c6e98..2aa865a1f 100644 --- a/field.go +++ b/field.go @@ -1884,7 +1884,7 @@ func applyDefaultOptions(o *FieldOptions) FieldOptions { // are included. func (o *FieldOptions) MarshalJSON() ([]byte, error) { switch o.Type { - case FieldTypeSet: + case FieldTypeSet, "": return json.Marshal(struct { Type string `json:"type"` CacheType string `json:"cacheType"` @@ -1975,7 +1975,7 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { o.Type, }) } - return nil, errors.New("invalid field type") + return nil, errors.Errorf("invalid field type: '%s'", o.Type) } // MinTimestamp returns the minimum value for a timestamp field.