Review skipped tests, and try to execute them again.

Signed-off-by: Antonio Navarro Perez <antnavper@gmail.com>
This commit is contained in:
Antonio Navarro Perez 2021-03-30 13:12:41 +02:00
parent 71b2cb80f7
commit 03659d8d39
10 changed files with 6 additions and 487 deletions

View file

@ -27,8 +27,6 @@ import (
)
func TestDiagnosticsClient(t *testing.T) {
t.Skip("does a listen on :0, skip for now. TODO(jea) restore this.")
// Mock server.
server := httptest.NewServer(nil)
defer server.Close()
@ -114,8 +112,6 @@ func TestDiagnosticsVersion_Compare(t *testing.T) {
}
func TestDiagnosticsVersion_Check(t *testing.T) {
t.Skip("does a listen on :0, skip for now. TODO(jea) restore this.")
// Mock server.
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)

View file

@ -32,7 +32,6 @@ import (
func TestHolder_Open(t *testing.T) {
t.Run("ErrIndexPermission", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
@ -41,7 +40,7 @@ func TestHolder_Open(t *testing.T) {
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
} else if err := h.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(h.IndexPath("test"), 0000); err != nil {
t.Fatal(err)
@ -55,13 +54,12 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
h := test.MustOpenHolder(t)
defer h.Close()
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
} else if err := h.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.IndexPath("test"), pilosa.ColumnAttrsFileName), 2); err != nil {
t.Fatal(err)
@ -71,81 +69,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFieldPermission", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
h := test.MustOpenHolder(t)
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path(), "foo", "bar"), 0000); err != nil {
t.Fatal(err)
}
defer func() {
_ = os.Chmod(filepath.Join(h.Path(), "foo", "bar"), 0755)
}()
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
h := test.MustOpenHolder(t)
defer h.Close()
var idx *pilosa.Index
var err error
if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", ".meta"), 2); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=loading meta: unmarshaling: unexpected EOF") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
h := test.MustOpenHolder(t)
defer h.Close()
var idx *pilosa.Index
var err error
if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", pilosa.RowAttrsFileName), 2); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=opening attrstore: opening storage: invalid database") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
roaringOnlyTest(t)
if os.Geteuid() == 0 {
@ -183,7 +107,6 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
roaringOnlyTest(t)
h := test.MustOpenHolder(t)

View file

@ -30,9 +30,8 @@ func TestTranslateStore_EntryReader(t *testing.T) {
// Ensure client can connect and stream the translate store data.
t.Run("OK", func(t *testing.T) {
t.Run("ServerDisconnect", func(t *testing.T) {
// This test is currently flawed, breaking intermittently with message:
// "translator_test.go:65: unexpected EOF"
t.Skip()
// This test is currently flawed
t.Skip("failing with error: invalid memory address or nil pointer dereference when reading entry")
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
@ -77,80 +76,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
t.Fatal(err)
}
})
/*
// Ensure server closes store reader if client disconnects.
t.Run("ClientDisconnect", func(t *testing.T) {
t.Skip() // can't mock server from http package
// Setup mock so that Read() hangs.
done := make(chan struct{})
var mrc mock.ReadCloser
mrc.ReadFunc = func(p []byte) (int, error) {
<-done
return 0, io.EOF
}
closeInvoked := make(chan struct{})
mrc.CloseFunc = func() error {
close(closeInvoked)
return nil
}
var translateStore mock.TranslateStore
translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
return &mrc, nil
}
opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
cluster := test.MustRunCluster(t, 1, []server.CommandOption{opts})
defer cluster.Close()
primary := cluster[0]
defer close(done)
// Connect to server and begin streaming.
ctx, cancel := context.WithCancel(context.Background())
store := http.NewTranslateStore(primary.URL())
if _, err := store.Reader(ctx, 0); err != nil {
t.Fatal(err)
}
// Cancel the context and check if server is closed.
cancel()
select {
case <-time.NewTimer(time.Millisecond * 100).C:
t.Fatal("expected server close")
case <-closeInvoked:
return
}
})
*/
})
/*
// Ensure client is notified if the server doesn't support streaming replication.
t.Run("ErrNotImplemented", func(t *testing.T) {
t.Skip() // can't mock server from http package
var translateStore mock.TranslateStore
translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
return nil, pilosa.ErrNotImplemented
}
opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
cluster := test.MustRunCluster(t, 1, []server.CommandOption{opts})
defer cluster.Close()
primary := cluster[0]
ts := http.NewTranslateStore(primary.URL())
_, err := ts.Reader(context.Background(), 0)
if err != pilosa.ErrNotImplemented {
t.Fatalf("unexpected error: %s", err)
}
})
*/
}
func benchmarkSetup(b *testing.B, ctx context.Context, key string, nkeys int) (string, pilosa.TranslateOffsetMap, func()) {

View file

@ -28,7 +28,7 @@ import (
func TestClusterStuff(t *testing.T) {
if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" {
t.Skip()
t.Skip("pilosa cluster tests are not enabled")
}
cli1, err := picli.NewInternalClient("pilosa1:10101", picli.GetHTTPClient(nil))
if err != nil {

View file

@ -1,31 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rbf
import (
"testing"
)
// This function exists to mark debugging helper function as "used" by the linter.
func TestUsed(t *testing.T) {
t.Skip("This function is always skipped")
/* dump(nil)
hexdump(nil)
pagedumpi(nil, "", nil)
treedump(nil, 0, "", nil)
onpanic(nil)
itohex(0)
*/
}

View file

@ -96,7 +96,6 @@ func TestTx_CommitRollback(t *testing.T) {
})
t.Run("SingleWriter", func(t *testing.T) {
//t.Skip("NEED TO FIX IN RACE") //TODO (twg)
db := MustOpenDB(t)
defer MustCloseDB(t, db)

View file

@ -915,97 +915,6 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) {
}
}
func TestClusterQueriesAfterRestart(t *testing.T) {
t.Skip("won't work on etcd since the node goes down and up but etcd old nodes won't know how to contact the restarted one.")
cluster := test.MustRunCluster(t, 3)
defer cluster.Close()
cmd1 := cluster.GetNode(1)
err := cmd1.AwaitState(disco.ClusterStateNormal, 100*time.Millisecond)
if err != nil {
t.Fatalf("starting cluster: %v", err)
}
for _, com := range cluster.Nodes {
nodes := com.API.Hosts(context.Background())
for _, n := range nodes {
if n.State != "READY" {
t.Fatalf("unexpected node state after upping cluster: %v", nodes)
}
}
}
cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{})
cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10))
// build a query to set the first bit in 100 shards
query := strings.Builder{}
for i := 0; i < 100; i++ {
query.WriteString(fmt.Sprintf("Set(%d, testfield=0)", i*pilosa.ShardWidth))
}
_, err = cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
Index: "testidx",
Query: query.String(),
})
if err != nil {
t.Fatalf("setting 100 bits in 100 shards: %v", err)
}
results, err := cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
Index: "testidx",
Query: "Count(Row(testfield=0))",
})
if err != nil {
t.Fatalf("counting row: %v", err)
}
if results.Results[0].(uint64) != 100 {
t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0])
}
err = cmd1.Command.Close()
if err != nil {
t.Fatalf("closing node1: %v", err)
}
// confirm that cluster stops accepting queries after one node closes
if _, err := cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
}
// Create new main with the same config.
config := cmd1.Command.Config
config.Bind = cmd1.API.Node().URI.HostPort()
cmd1.Command = server.NewCommand(cmd1.Stdin, cmd1.Stdout, cmd1.Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
cmd1.Command.Config = config
err = cmd1.Start()
if err != nil {
t.Fatalf("reopening node 1: %v", err)
}
state1, err1 := cmd1.API.State()
if err1 != nil {
t.Fatalf("getting state foor node 1: %v", err)
}
for state1 != disco.ClusterStateNormal {
time.Sleep(time.Millisecond)
}
results, err = cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
Index: "testidx",
Query: "Count(Row(testfield=0))",
})
if err != nil {
t.Fatalf("counting row: %v", err)
}
if results.Results[0].(uint64) != 100 {
t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0])
}
}
// TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address.
func TestClusterExhaustingConnections(t *testing.T) {
if !runStress {
t.Skip("stress")

View file

@ -454,7 +454,7 @@ func CheckClusterState(m *Command, state disco.ClusterState, n int) bool {
if apiState == state {
return true
}
time.Sleep(10 * time.Millisecond)
time.Sleep(100 * time.Millisecond)
}
return false
}

View file

@ -131,20 +131,6 @@ func (m *Command) Reopen() error {
return m.Start()
}
// SoftOpen is like Reopen, but doesn't close the program first.
// This is useful in the case where a test needs to decouple the
// close from the re-open (for example, there may need to be
// actions which take place between those two steps).
func (m *Command) SoftOpen() error {
// Create new main with the same config.
config := m.Command.Config
m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, m.commandOptions...)
m.Command.Config = config
// Run new program.
return m.Start()
}
// MustCreateIndex uses this command's API to create an index and fails the test
// if there is an error.
func (m *Command) MustCreateIndex(tb testing.TB, name string, opts pilosa.IndexOptions) *pilosa.Index {

View file

@ -26,7 +26,6 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/disco"
"github.com/pilosa/pilosa/v2/http"
"github.com/pilosa/pilosa/v2/mock"
"github.com/pilosa/pilosa/v2/server"
@ -189,109 +188,6 @@ func TestMultiTranslateEntryReader(t *testing.T) {
})
}
// Test key translation with multiple nodes.
func TestTranslation_Reset(t *testing.T) {
// We need to ensure that the translate key partitions for each
// node are getting set as read-only based on the full cluster,
// not just the state of the cluster at the time of the individual
// node restart.
t.Run("RollingRestart", func(t *testing.T) {
t.Skip("skipping because disco needs asynchrounous restart")
// Start a 4-node cluster.
// Note that the prefix on the nodeID is intentional; it puts the
// nodes in a specific order which exercises the condition for
// which we are testing. In a normal use case, these would be
// randomly generated uuids, so this is mimicking that.
c := test.MustRunCluster(t, 4,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("2node0"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("4node1"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("3node2"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("1node3"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
node0 := c.GetNode(0)
node1 := c.GetNode(1)
node2 := c.GetNode(2)
node3 := c.GetNode(3)
ctx := context.Background()
idx := "i"
// Create an index with keys.
if _, err := node0.API.CreateIndex(ctx, idx,
pilosa.IndexOptions{
Keys: true,
}); err != nil {
t.Fatal(err)
}
// Stop the cluster.
if err := node0.Command.Close(); err != nil {
t.Fatal(err)
}
if err := node1.Command.Close(); err != nil {
t.Fatal(err)
}
if err := node2.Command.Close(); err != nil {
t.Fatal(err)
}
if err := node3.Command.Close(); err != nil {
t.Fatal(err)
}
// Restart the nodes serially.
if err := node0.SoftOpen(); err != nil {
t.Fatal(err)
}
if err := node1.SoftOpen(); err != nil {
t.Fatal(err)
}
if err := node2.SoftOpen(); err != nil {
t.Fatal(err)
}
if err := node3.SoftOpen(); err != nil {
t.Fatal(err)
}
// Send a key translation request that triggers
// a read-only translate store error if the
// translate store sync is not reset correctly.
// Generate request body for translate row keys request
reqBody, err := node0.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
Index: idx,
Keys: []string{"a1"},
})
if err != nil {
t.Fatal(err)
}
if _, err := node0.API.TranslateKeys(ctx, bytes.NewReader(reqBody)); err != nil {
t.Fatal(err)
}
})
}
func TestTranslation_KeyNotFound(t *testing.T) {
c := test.MustRunCluster(t, 4,
[]server.CommandOption{
@ -443,96 +339,11 @@ func TestInMemTranslateStore_ReadKey(t *testing.T) {
}
// Test index key translation replication under node failure.
func TestTranslation_Replication(t *testing.T) {
t.Run("Replication", func(t *testing.T) {
t.Skip("this test is fragile and doesn't work with randomly ordered nodes. it also seems to assume failover for index key partitions, which does not exist")
c := test.MustRunCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(2),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(2),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(2),
)},
)
defer c.Close()
coord := c.GetPrimary()
other := c.GetNonPrimary()
ctx := context.Background()
idx := "i"
field := "f"
// Create an index with keys.
if _, err := coord.API.CreateIndex(ctx, idx,
pilosa.IndexOptions{
Keys: true,
}); err != nil {
t.Fatal(err)
}
if _, err := coord.API.CreateField(ctx, idx, field); err != nil {
t.Fatal(err)
}
// Write data on first node.
// these keys are a minimal example to reproduce the problem for the case of a 3-node cluster with replication factor 2
if _, err := coord.Queryf(t, idx, "", `
Set("x1", f=1)
Set("x2", f=1)
`); err != nil {
t.Fatal(err)
}
exp := `{"results":[{"attrs":{},"columns":[],"keys":["x1","x2"]}]}`
coordState, err := coord.API.State()
if err != nil || !test.CheckClusterState(coord, disco.ClusterStateNormal, 1000) {
t.Fatalf("unexpected coord cluster state: %s, got: %s, err: %v", disco.ClusterStateNormal, coordState, err)
}
otherState, err := other.API.State()
if err != nil || !test.CheckClusterState(other, disco.ClusterStateNormal, 1000) {
t.Fatalf("unexpected other cluster state: %s, got: %s, err: %v", disco.ClusterStateNormal, otherState, err)
}
// Verify the data exists
coord.QueryExpect(t, idx, "", `Row(f=1)`, exp)
// Kill a non-primary node.
if err := c.CloseAndRemoveNonPrimary(); err != nil {
t.Fatal(err)
}
coordState, err = coord.API.State()
if err != nil || !test.CheckClusterState(coord, disco.ClusterStateDegraded, 1000) {
t.Fatalf("unexpected coord cluster state: %s, got: %s", disco.ClusterStateDegraded, coordState)
}
// Verify the data exists with one node down
coord.QueryExpect(t, idx, "", `Row(f=1)`, exp)
})
}
// Test key translation with multiple nodes.
func TestTranslation_Primary(t *testing.T) {
// Ensure that field key translations requests sent to
// non-primary nodes are forwarded to the primary.
t.Run("ForwardFieldKey", func(t *testing.T) {
t.Skip("Short term skip to avoid go 1.13 test Should remove ASAP")
// Start a 2-node cluster.
c := test.MustRunCluster(t, 3,
[]server.CommandOption{