mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
add tests to cover the coordinator logic for multi-node clusters
This commit is contained in:
Travis Turner
parent
fd96d3a02e
commit
0a6f0e92d8
2 changed files with 185 additions and 85 deletions
|
|
@ -101,6 +101,33 @@ func TestImportCommand_RunValue(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure that import with keys runs.
|
||||
func TestImportCommand_RunKeys(t *testing.T) {
|
||||
buf := bytes.Buffer{}
|
||||
stdin, stdout, stderr := GetIO(buf)
|
||||
cm := NewImportCommand(stdin, stdout, stderr)
|
||||
file, err := ioutil.TempFile("", "import-key.csv")
|
||||
file.Write([]byte("foo1,bar2\nfoo3,bar4\nfoo5,bar6"))
|
||||
ctx := context.Background()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cmd := test.MustRunCluster(t, 1)[0]
|
||||
cm.Host = cmd.API.Node().URI.HostPort()
|
||||
|
||||
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
|
||||
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`)))
|
||||
|
||||
cm.Index = "i"
|
||||
cm.Field = "f"
|
||||
cm.Paths = []string{file.Name()}
|
||||
err = cm.Run(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Import Run with keys doesn't work: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCommand_InvalidFile(t *testing.T) {
|
||||
cmd := test.MustRunCluster(t, 1)[0]
|
||||
|
||||
|
|
|
|||
|
|
@ -204,100 +204,173 @@ func TestClient_Import(t *testing.T) {
|
|||
|
||||
// Ensure client can bulk import data.
|
||||
func TestClient_ImportKeys(t *testing.T) {
|
||||
cmd := test.MustRunCluster(t, 1)[0]
|
||||
host := cmd.URL()
|
||||
t.Run("SingleNode", func(t *testing.T) {
|
||||
cmd := test.MustRunCluster(t, 1)[0]
|
||||
host := cmd.URL()
|
||||
|
||||
cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true})
|
||||
cmd.MustCreateIndex(t, "unkeyed", pilosa.IndexOptions{Keys: false})
|
||||
cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true})
|
||||
cmd.MustCreateIndex(t, "unkeyed", pilosa.IndexOptions{Keys: false})
|
||||
|
||||
cmd.MustCreateField(t, "keyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
|
||||
cmd.MustCreateField(t, "keyed", "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000))
|
||||
cmd.MustCreateField(t, "unkeyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
|
||||
cmd.MustCreateField(t, "keyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
|
||||
cmd.MustCreateField(t, "keyed", "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000))
|
||||
cmd.MustCreateField(t, "unkeyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
|
||||
|
||||
// Send import request.
|
||||
c := MustNewClient(host, http.GetHTTPClient(nil))
|
||||
// Send import request.
|
||||
c := MustNewClient(host, http.GetHTTPClient(nil))
|
||||
|
||||
t.Run("Import keyed,keyed", func(t *testing.T) {
|
||||
if err := c.Import(context.Background(), "keyed", "keyedf", 0, []pilosa.Bit{
|
||||
{RowKey: "green", ColumnKey: "eve"},
|
||||
{RowKey: "green", ColumnKey: "alice"},
|
||||
{RowKey: "green", ColumnKey: "bob"},
|
||||
{RowKey: "blue", ColumnKey: "eve"},
|
||||
{RowKey: "blue", ColumnKey: "alice"},
|
||||
{RowKey: "purple", ColumnKey: "eve"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd.MustRecalculateCaches(t)
|
||||
resp := cmd.MustQuery(t, &pilosa.QueryRequest{
|
||||
Index: "keyed",
|
||||
Query: "TopN(keyedf)",
|
||||
t.Run("Import keyed,keyed", func(t *testing.T) {
|
||||
if err := c.Import(context.Background(), "keyed", "keyedf", 0, []pilosa.Bit{
|
||||
{RowKey: "green", ColumnKey: "eve"},
|
||||
{RowKey: "green", ColumnKey: "alice"},
|
||||
{RowKey: "green", ColumnKey: "bob"},
|
||||
{RowKey: "blue", ColumnKey: "eve"},
|
||||
{RowKey: "blue", ColumnKey: "alice"},
|
||||
{RowKey: "purple", ColumnKey: "eve"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd.MustRecalculateCaches(t)
|
||||
resp := cmd.MustQuery(t, &pilosa.QueryRequest{
|
||||
Index: "keyed",
|
||||
Query: "TopN(keyedf)",
|
||||
})
|
||||
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
|
||||
t.Fatal("unexpected response type %T", resp.Results[0])
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
{Key: "green", Count: 3},
|
||||
{Key: "blue", Count: 2},
|
||||
{Key: "purple", Count: 1},
|
||||
}) {
|
||||
t.Fatalf("unexpected topn result: %v", pairs)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Import keyed,unkeyedf", func(t *testing.T) {
|
||||
if err := c.Import(context.Background(), "keyed", "unkeyedf", 0, []pilosa.Bit{
|
||||
{RowID: 1, ColumnKey: "eve"},
|
||||
{RowID: 1, ColumnKey: "alice"},
|
||||
{RowID: 1, ColumnKey: "bob"},
|
||||
{RowID: 2, ColumnKey: "eve"},
|
||||
{RowID: 2, ColumnKey: "alice"},
|
||||
{RowID: 3, ColumnKey: "eve"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd.MustRecalculateCaches(t)
|
||||
resp := cmd.MustQuery(t, &pilosa.QueryRequest{
|
||||
Index: "keyed",
|
||||
Query: "TopN(unkeyedf)",
|
||||
})
|
||||
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
|
||||
t.Fatal("unexpected response type %T", resp.Results[0])
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
{ID: 1, Count: 3},
|
||||
{ID: 2, Count: 2},
|
||||
{ID: 3, Count: 1},
|
||||
}) {
|
||||
t.Fatalf("unexpected topn result: %v", pairs)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Import unkeyed,keyed", func(t *testing.T) {
|
||||
if err := c.Import(context.Background(), "unkeyed", "keyedf", 0, []pilosa.Bit{
|
||||
{RowKey: "green", ColumnID: 1},
|
||||
{RowKey: "green", ColumnID: 2},
|
||||
{RowKey: "green", ColumnID: 3},
|
||||
{RowKey: "blue", ColumnID: 1},
|
||||
{RowKey: "blue", ColumnID: 2},
|
||||
{RowKey: "purple", ColumnID: 1},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd.MustRecalculateCaches(t)
|
||||
resp := cmd.MustQuery(t, &pilosa.QueryRequest{
|
||||
Index: "unkeyed",
|
||||
Query: "TopN(keyedf)",
|
||||
})
|
||||
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
|
||||
t.Fatal("unexpected response type %T", resp.Results[0])
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
{Key: "green", Count: 3},
|
||||
{Key: "blue", Count: 2},
|
||||
{Key: "purple", Count: 1},
|
||||
}) {
|
||||
t.Fatalf("unexpected topn result: %v", pairs)
|
||||
}
|
||||
})
|
||||
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
|
||||
t.Fatal("unexpected response type %T", resp.Results[0])
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
{Key: "green", Count: 3},
|
||||
{Key: "blue", Count: 2},
|
||||
{Key: "purple", Count: 1},
|
||||
}) {
|
||||
t.Fatalf("unexpected topn result: %v", pairs)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Import keyed,unkeyedf", func(t *testing.T) {
|
||||
if err := c.Import(context.Background(), "keyed", "unkeyedf", 0, []pilosa.Bit{
|
||||
{RowID: 1, ColumnKey: "eve"},
|
||||
{RowID: 1, ColumnKey: "alice"},
|
||||
{RowID: 1, ColumnKey: "bob"},
|
||||
{RowID: 2, ColumnKey: "eve"},
|
||||
{RowID: 2, ColumnKey: "alice"},
|
||||
{RowID: 3, ColumnKey: "eve"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd.MustRecalculateCaches(t)
|
||||
resp := cmd.MustQuery(t, &pilosa.QueryRequest{
|
||||
Index: "keyed",
|
||||
Query: "TopN(unkeyedf)",
|
||||
})
|
||||
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
|
||||
t.Fatal("unexpected response type %T", resp.Results[0])
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
{ID: 1, Count: 3},
|
||||
{ID: 2, Count: 2},
|
||||
{ID: 3, Count: 1},
|
||||
}) {
|
||||
t.Fatalf("unexpected topn result: %v", pairs)
|
||||
}
|
||||
})
|
||||
t.Run("MultiNode", func(t *testing.T) {
|
||||
cluster := test.MustRunCluster(t, 2)
|
||||
cmd0 := cluster[0]
|
||||
cmd1 := cluster[1]
|
||||
host0 := cmd0.URL()
|
||||
host1 := cmd1.URL()
|
||||
|
||||
t.Run("Import unkeyed,keyed", func(t *testing.T) {
|
||||
if err := c.Import(context.Background(), "unkeyed", "keyedf", 0, []pilosa.Bit{
|
||||
{RowKey: "green", ColumnID: 1},
|
||||
{RowKey: "green", ColumnID: 2},
|
||||
{RowKey: "green", ColumnID: 3},
|
||||
{RowKey: "blue", ColumnID: 1},
|
||||
{RowKey: "blue", ColumnID: 2},
|
||||
{RowKey: "purple", ColumnID: 1},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd.MustRecalculateCaches(t)
|
||||
resp := cmd.MustQuery(t, &pilosa.QueryRequest{
|
||||
Index: "unkeyed",
|
||||
Query: "TopN(keyedf)",
|
||||
})
|
||||
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
|
||||
t.Fatal("unexpected response type %T", resp.Results[0])
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
{Key: "green", Count: 3},
|
||||
{Key: "blue", Count: 2},
|
||||
{Key: "purple", Count: 1},
|
||||
}) {
|
||||
t.Fatalf("unexpected topn result: %v", pairs)
|
||||
}
|
||||
})
|
||||
cmd0.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true})
|
||||
cmd0.MustCreateField(t, "keyed", "keyedf0", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
|
||||
cmd0.MustCreateField(t, "keyed", "keyedf1", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
|
||||
|
||||
// Send import request.
|
||||
c0 := MustNewClient(host0, http.GetHTTPClient(nil))
|
||||
c1 := MustNewClient(host1, http.GetHTTPClient(nil))
|
||||
|
||||
// Import to node0.
|
||||
t.Run("Import node0", func(t *testing.T) {
|
||||
if err := c0.ImportK(context.Background(), "keyed", "keyedf0", []pilosa.Bit{
|
||||
{RowKey: "green", ColumnKey: "eve"},
|
||||
{RowKey: "green", ColumnKey: "alice"},
|
||||
{RowKey: "green", ColumnKey: "bob"},
|
||||
{RowKey: "blue", ColumnKey: "eve"},
|
||||
{RowKey: "blue", ColumnKey: "alice"},
|
||||
{RowKey: "purple", ColumnKey: "eve"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd0.MustRecalculateCaches(t)
|
||||
resp := cmd0.MustQuery(t, &pilosa.QueryRequest{
|
||||
Index: "keyed",
|
||||
Query: "TopN(keyedf0)",
|
||||
})
|
||||
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
|
||||
t.Fatal("unexpected response type %T", resp.Results[0])
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
{Key: "green", Count: 3},
|
||||
{Key: "blue", Count: 2},
|
||||
{Key: "purple", Count: 1},
|
||||
}) {
|
||||
t.Fatalf("unexpected topn result: %v", pairs)
|
||||
}
|
||||
})
|
||||
|
||||
// Import to node1 (ensure import is routed to coordinator for translation).
|
||||
t.Run("Import node1", func(t *testing.T) {
|
||||
if err := c1.ImportK(context.Background(), "keyed", "keyedf1", []pilosa.Bit{
|
||||
{RowKey: "green", ColumnKey: "eve"},
|
||||
{RowKey: "green", ColumnKey: "alice"},
|
||||
{RowKey: "green", ColumnKey: "bob"},
|
||||
{RowKey: "blue", ColumnKey: "eve"},
|
||||
{RowKey: "blue", ColumnKey: "alice"},
|
||||
{RowKey: "purple", ColumnKey: "eve"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd1.MustRecalculateCaches(t)
|
||||
resp := cmd1.MustQuery(t, &pilosa.QueryRequest{
|
||||
Index: "keyed",
|
||||
Query: "TopN(keyedf1)",
|
||||
})
|
||||
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
|
||||
t.Fatal("unexpected response type %T", resp.Results[0])
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
{Key: "green", Count: 3},
|
||||
{Key: "blue", Count: 2},
|
||||
{Key: "purple", Count: 1},
|
||||
}) {
|
||||
t.Fatalf("unexpected topn result: %v", pairs)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// Ensure client can bulk import value data.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue