mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 00:55:55 +00:00
do a backup in a go test for coverage purposes
also found a weird issue with schema marshalling if you create a field thru the api w/o specifying a field type, you get slightly different behavior than going thru the HTTP handler which is... not ideal. I changed the marshaler to accept an empty field type.
This commit is contained in:
Matthew Jaffee
parent
3761fc6d3c
commit
aff3d3ddd9
3 changed files with 58 additions and 21 deletions
|
|
@ -18,7 +18,7 @@ import (
|
|||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// BackupCommand represents a command for backing up a Pilosa node.
|
||||
// BackupCommand represents a command for backing up a FeatureBase node.
|
||||
type BackupCommand struct { // nolint: maligned
|
||||
tlsConfig *tls.Config
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import (
|
|||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
pilosa "github.com/molecula/featurebase/v2"
|
||||
"github.com/molecula/featurebase/v2/boltdb"
|
||||
"github.com/molecula/featurebase/v2/ctl"
|
||||
"github.com/molecula/featurebase/v2/disco"
|
||||
"github.com/molecula/featurebase/v2/http"
|
||||
"github.com/molecula/featurebase/v2/pql"
|
||||
|
|
@ -7015,11 +7016,16 @@ func TestMissingKeyRegression(t *testing.T) {
|
|||
// (single and multi-node clusters, different endpoints for the
|
||||
// queries (HTTP, GRPC, Postgres), etc.).
|
||||
func TestVariousQueries(t *testing.T) {
|
||||
for _, clusterSize := range []int{1, 3, 4, 7} {
|
||||
for _, clusterSize := range []int{1, 3, 7} {
|
||||
clusterSize := clusterSize
|
||||
t.Run(fmt.Sprintf("%d-node", clusterSize), func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, clusterSize)
|
||||
defer c.Close()
|
||||
|
||||
// put a variety of data into the cluster
|
||||
populateTestData(t, c)
|
||||
backupTest(t, c)
|
||||
|
||||
variousQueries(t, c)
|
||||
variousQueriesOnTimeFields(t, c)
|
||||
variousQueriesOnPercentiles(t, c)
|
||||
|
|
@ -7028,6 +7034,30 @@ func TestVariousQueries(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func backupTest(t *testing.T, c *test.Cluster) {
|
||||
// should this really be in executor? No. But all these
|
||||
// integration-y query tests probably shouldn't be either. My goal
|
||||
// putting this here is to take advantage of already-existing
|
||||
// clusters and data.
|
||||
|
||||
td, err := testhook.TempDir(t, "backupTest")
|
||||
if err != nil {
|
||||
t.Fatalf("can't even get a temp dir, what a ripoff: %v", err)
|
||||
}
|
||||
td = td + "/backupTest"
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
backupCommand := ctl.NewBackupCommand(nil, buf, buf)
|
||||
backupCommand.Host = c.Nodes[len(c.Nodes)-1].URL() // don't pick node 0 so we don't always get primary (better code coverage)
|
||||
backupCommand.Index = usersIndex
|
||||
backupCommand.OutputDir = td
|
||||
|
||||
if err := backupCommand.Run(context.Background()); err != nil {
|
||||
t.Log(buf.String())
|
||||
t.Fatalf("running backup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// tests for abbreviating time values in queries
|
||||
func variousQueriesOnPercentiles(t *testing.T, c *test.Cluster) {
|
||||
// todo, make rand more random, 42 isnt the answer to everything
|
||||
|
|
@ -7332,10 +7362,12 @@ func variousQueriesOnTimeFields(t *testing.T, c *test.Cluster) {
|
|||
}
|
||||
}
|
||||
|
||||
func variousQueries(t *testing.T, c *test.Cluster) {
|
||||
var usersIndex = "users"
|
||||
|
||||
func populateTestData(t *testing.T, c *test.Cluster) {
|
||||
// Create and populate "likenums" similar to "likes", but without keys on the field.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums")
|
||||
c.ImportIDKey(t, "users", "likenums", []test.KeyID{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums")
|
||||
c.ImportIDKey(t, usersIndex, "likenums", []test.KeyID{
|
||||
{ID: 1, Key: "userA"},
|
||||
{ID: 2, Key: "userB"},
|
||||
{ID: 3, Key: "userC"},
|
||||
|
|
@ -7353,8 +7385,8 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
})
|
||||
|
||||
// Create and populate "likes" field.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys())
|
||||
c.ImportKeyKey(t, "users", "likes", [][2]string{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys())
|
||||
c.ImportKeyKey(t, usersIndex, "likes", [][2]string{
|
||||
{"molecula", "userA"},
|
||||
{"pilosa", "userB"},
|
||||
{"pangolin", "userC"},
|
||||
|
|
@ -7370,8 +7402,8 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
})
|
||||
|
||||
// Create and populate "dinner" field.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "dinner", pilosa.OptFieldKeys())
|
||||
c.ImportKeyKey(t, "users", "dinner", [][2]string{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "dinner", pilosa.OptFieldKeys())
|
||||
c.ImportKeyKey(t, usersIndex, "dinner", [][2]string{
|
||||
{"leftovers", "userB"},
|
||||
{"pizza", "userA"},
|
||||
{"pizza", "userB"},
|
||||
|
|
@ -7381,11 +7413,11 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
})
|
||||
|
||||
// Create and populate "places_visited" time field.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM")))
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM")))
|
||||
ts2019Jan01 := int64(1546300800) * 1e+9 // 2019 January 1st 0:00:00
|
||||
ts2019Aug01 := int64(1564617600) * 1e+9 // 2019 August 1st 0:00:00
|
||||
ts2020Jan01 := int64(1577836800) * 1e+9 // 2020 January 1st 0:00:00
|
||||
c.ImportTimeQuantumKey(t, "users", "places_visited", []test.TimeQuantumKey{
|
||||
c.ImportTimeQuantumKey(t, usersIndex, "places_visited", []test.TimeQuantumKey{
|
||||
// 2019 January: nairobi, paris, austin, toronto
|
||||
{RowKey: "nairobi", ColKey: "userB", Ts: ts2019Jan01},
|
||||
{RowKey: "paris", ColKey: "userC", Ts: ts2019Jan01},
|
||||
|
|
@ -7405,8 +7437,8 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
})
|
||||
|
||||
// Create and populate "affinity" int field with negative, positive, zero and null values.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "affinity", pilosa.OptFieldTypeInt(-1000, 1000))
|
||||
c.ImportIntKey(t, "users", "affinity", []test.IntKey{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "affinity", pilosa.OptFieldTypeInt(-1000, 1000))
|
||||
c.ImportIntKey(t, usersIndex, "affinity", []test.IntKey{
|
||||
{Val: 10, Key: "userA"},
|
||||
{Val: -10, Key: "userB"},
|
||||
{Val: 5, Key: "userC"},
|
||||
|
|
@ -7415,8 +7447,8 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
})
|
||||
|
||||
// Create and populate "net_worth" int field with positive values.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "net_worth", pilosa.OptFieldTypeInt(-100000000, 100000000))
|
||||
c.ImportIntKey(t, "users", "net_worth", []test.IntKey{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "net_worth", pilosa.OptFieldTypeInt(-100000000, 100000000))
|
||||
c.ImportIntKey(t, usersIndex, "net_worth", []test.IntKey{
|
||||
{Val: 1, Key: "userA"},
|
||||
{Val: 10, Key: "userB"},
|
||||
{Val: 100, Key: "userC"},
|
||||
|
|
@ -7425,8 +7457,8 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
{Val: 100000, Key: "userF"},
|
||||
})
|
||||
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "zip_code", pilosa.OptFieldTypeInt(0, 100000))
|
||||
c.ImportIntKey(t, "users", "zip_code", []test.IntKey{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "zip_code", pilosa.OptFieldTypeInt(0, 100000))
|
||||
c.ImportIntKey(t, usersIndex, "zip_code", []test.IntKey{
|
||||
{Val: 78739, Key: "userA"},
|
||||
{Val: 78739, Key: "userB"},
|
||||
{Val: 19707, Key: "userC"},
|
||||
|
|
@ -7434,7 +7466,12 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
{Val: 86753, Key: "userE"},
|
||||
{Val: 78739, Key: "userG"},
|
||||
})
|
||||
}
|
||||
|
||||
func variousQueries(t *testing.T, c *test.Cluster) {
|
||||
// NOTE: this relies on populateTestData being called first
|
||||
|
||||
// define and run a bunch of tests
|
||||
tests := []struct {
|
||||
query string
|
||||
qrVerifier func(t *testing.T, resp pilosa.QueryResponse)
|
||||
|
|
@ -7781,8 +7818,8 @@ leftovers,1
|
|||
|
||||
for i, tst := range tests {
|
||||
t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) {
|
||||
resp := c.Query(t, "users", tst.query)
|
||||
tr := c.QueryGRPC(t, "users", tst.query)
|
||||
resp := c.Query(t, usersIndex, tst.query)
|
||||
tr := c.QueryGRPC(t, usersIndex, tst.query)
|
||||
if tst.qrVerifier != nil {
|
||||
tst.qrVerifier(t, resp)
|
||||
}
|
||||
|
|
|
|||
4
field.go
4
field.go
|
|
@ -1884,7 +1884,7 @@ func applyDefaultOptions(o *FieldOptions) FieldOptions {
|
|||
// are included.
|
||||
func (o *FieldOptions) MarshalJSON() ([]byte, error) {
|
||||
switch o.Type {
|
||||
case FieldTypeSet:
|
||||
case FieldTypeSet, "":
|
||||
return json.Marshal(struct {
|
||||
Type string `json:"type"`
|
||||
CacheType string `json:"cacheType"`
|
||||
|
|
@ -1975,7 +1975,7 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
|
|||
o.Type,
|
||||
})
|
||||
}
|
||||
return nil, errors.New("invalid field type")
|
||||
return nil, errors.Errorf("invalid field type: '%s'", o.Type)
|
||||
}
|
||||
|
||||
// MinTimestamp returns the minimum value for a timestamp field.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue