scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor testing to share clusters more often

Browse source 
When doing tests, we create a ton of one-off clusters. This
turns out to be expensive and slow. Fixing it is surprisingly hard.

Fundamentally: If we're sharing clusters, we need to use different
indexes for each test, to avoid clashes. This changes index names.
As a side-effect, this reorders many partition-based things, like
the order keys are returned in. Thus, to fix this, we change a lot
of tests to no longer depend on the *order* in which strings are
returned.

Having done that, we can also discard the ModHasher behavior, since
that only existed to allow us to reliably predict partitioning.

The basic design is as follows: Instead of a cluster being a
[]*Command, a "shareable" cluster is now a []*Command plus some
flags, and a "cluster" is a pointer to a possibly-shared cluster,
plus a link to the specific test using this specific cluster,
and correspondingly, its test name suitably coerced to be a valid
index name prefix.

The "test.Cluster" object now has methods to allow retrieving an
index name, and also implemnts fmt.Formatter to let you use,
e.g., `%i` with it in Sprintf to get "the index name, plus an i".
(This works for everything but %p and %T.)

This allows us to consistently rework all the many things that
use index names in a persistent way.

We also have `MustUnshared` and `MustRunUnsharedCluster` methods
which allow us to specify that a given test needs its own cluster
for some reason. For instance, the tests that want to run backups
need their own isolated cluster, and the tests that want to close
or reopen nodes need their own cluster because a reopened cluster
won't have working GRPC for some reason.

On "closing" a shared cluster (actually the test-specific wrapper
that reflects a given sharing), we delete any indexes starting with
that test's index name prefix. Otherwise, the huge pile of open
indexes prevents `go test -race` from working on MacOS, where we
run out of address space too quickly.

This is fairly enormous but most of the individual changes are
fairly trivial things like replacing the string "i" with "c.Idx()".

We also tweaked a test that failed for me a couple of times to
not depend on sort order.
This commit is contained in:
Seebs 2022-08-08 15:50:44 -05:00 • committed by Kasey Rodgers
parent 799e70fe46
commit f6d17b1b58
27 changed files with 1909 additions and 1648 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

215
api_test.go
View file

@ -23,48 +23,25 @@ import (
"time"
"github.com/golang-jwt/jwt"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/authn"
"github.com/featurebasedb/featurebase/v3/boltdb"
"github.com/featurebasedb/featurebase/v3/roaring"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/shardwidth"
"github.com/featurebasedb/featurebase/v3/test"
. "github.com/featurebasedb/featurebase/v3/vprint" // nolint:staticcheck
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/shardwidth"
"github.com/molecula/featurebase/v3/test"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
"golang.org/x/sync/errgroup"
)
func TestAPI_Import(t *testing.T) {
c := test.MustRunCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(t, 3)
defer c.Close()
m0 := c.GetNode(0)
m1 := c.GetNode(1)
indexNames := map[bool]string{false: "i", true: "ki"}
indexNames := map[bool]string{false: c.Idx("u"), true: c.Idx("k")}
fieldNames := map[bool]string{false: "f", true: "kf"}
ctx := context.Background()
@ -104,7 +81,7 @@ func TestAPI_Import(t *testing.T) {
t.Run("RowIDColumnKey", func(t *testing.T) {
// Import data with keys to the primary and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
// translated and forwarded to the owner of shard 0
req := &pilosa.ImportRequest{
Index: indexNames[true],
Field: fieldNames[false],
@ -152,7 +129,7 @@ func TestAPI_Import(t *testing.T) {
}
})
t.Run("ExpectedErrors", func(t *testing.T) {
t.Skip() // skipping due to change partitioning strategy
t.Skip("partitioning strategy changed, test not supported") // skipping due to change partitioning strategy
ctx := context.Background()
for ik, indexName := range indexNames {
for fk, fieldName := range fieldNames {
@ -224,26 +201,7 @@ func TestAPI_Import(t *testing.T) {
}
func TestAPI_ImportValue(t *testing.T) {
c := test.MustRunCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(t, 3)
defer c.Close()
coord := c.GetPrimary()
@ -253,7 +211,7 @@ func TestAPI_ImportValue(t *testing.T) {
t.Run("ValColumnKey", func(t *testing.T) {
ctx := context.Background()
index := "valck"
index := c.Idx("valck")
field := "f"
_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true})
@ -275,7 +233,7 @@ func TestAPI_ImportValue(t *testing.T) {
colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
// Import data with keys to the primary and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
// translated and forwarded to the owner of shard 0
req := &pilosa.ImportValueRequest{
Index: index,
Field: field,
@ -293,18 +251,24 @@ func TestAPI_ImportValue(t *testing.T) {
pql := fmt.Sprintf("Row(%s>0)", field)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
if err != nil {
t.Fatal(err)
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
}
keys := res.Results[0].(*pilosa.Row).Keys
if !sameStringSlice(keys, colKeys) {
t.Fatalf("unexpected column keys: %+v", keys)
}
// Query node1.
if err := test.RetryUntil(5*time.Second, func() error {
if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
return err
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
return fmt.Errorf("unexpected column keys: %+v", keys)
res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
if err != nil {
t.Fatal(err)
}
keys := res.Results[0].(*pilosa.Row).Keys
if !sameStringSlice(keys, colKeys) {
t.Fatalf("unexpected column keys: %+v", keys)
}
return nil
}); err != nil {
@ -314,7 +278,7 @@ func TestAPI_ImportValue(t *testing.T) {
t.Run("ValIntEmpty", func(t *testing.T) {
ctx := context.Background()
index := "valintempty"
index := c.Idx("valintempty")
field := "fld"
createIndexForTest(index, coord, t)
createFieldForTest(index, field, coord, t)
@ -387,7 +351,7 @@ func TestAPI_ImportValue(t *testing.T) {
colIDs = append(colIDs, uint64(i))
}
// Import data with keys to node1 and verify that it gets translated and
// forwarded to the owner of shard 0 (node0; because of offsetModHasher)
// forwarded to the owner of shard 0
req := &pilosa.ImportValueRequest{
Index: index,
Field: field,
@ -410,7 +374,7 @@ func TestAPI_ImportValue(t *testing.T) {
t.Run("ValDecimalFieldNegativeScale", func(t *testing.T) {
ctx := context.Background()
index := "valdecneg"
index := c.Idx("valdecneg")
field := "fdecneg"
_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
@ -424,9 +388,9 @@ func TestAPI_ImportValue(t *testing.T) {
})
t.Run("ValTimestampField", func(t *testing.T) {
t.Skip() // skipping due to change partitioning strategy
t.Skip("partition strategy change invalidated") // skipping due to change partitioning strategy
ctx := context.Background()
index := "valts"
index := c.Idx("valts")
field := "fts"
_, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
@ -447,7 +411,7 @@ func TestAPI_ImportValue(t *testing.T) {
}
// Import data with keys to node1 and verify that it gets translated and
// forwarded to the owner of shard 0 (node0; because of offsetModHasher)
// forwarded to the owner of shard 0
req := &pilosa.ImportValueRequest{
Index: index,
Field: field,
@ -472,9 +436,9 @@ func TestAPI_ImportValue(t *testing.T) {
})
t.Run("ValStringField", func(t *testing.T) {
t.Skip() // skipping due to change partitioning strategy
t.Skip("partition strategy change invalidated") // skipping due to change partitioning strategy
ctx := context.Background()
index := "valstr"
index := c.Idx("valstr")
field := "fstr"
fgnIndex := "fgnvalstr"
@ -506,8 +470,7 @@ func TestAPI_ImportValue(t *testing.T) {
}
// Import data with keys to the node0 and verify that it gets translated
// and forwarded to the owner of shard 0 (node1; because of
// offsetModHasher)
// and forwarded to the owner of shard 0
req := &pilosa.ImportValueRequest{
Index: index,
Field: field,
@ -534,17 +497,10 @@ func TestAPI_ImportValue(t *testing.T) {
func TestAPI_Ingest(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := test.MustRunCluster(t, 1,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(t, 1)
defer c.Close()
coord := c.GetPrimary()
index := "ingest"
index := c.Idx()
setField := "set"
timeField := "tq"
intField := "int"
@ -654,7 +610,7 @@ func TestAPI_Ingest(t *testing.T) {
},
},
}
if err := coord.API.ImportRoaringShard(context.Background(), "ingest", 8, request); err != nil {
if err := coord.API.ImportRoaringShard(context.Background(), c.Idx(), 8, request); err != nil {
t.Fatalf("ingesting: %v", err)
}
@ -666,19 +622,19 @@ func TestAPI_Ingest(t *testing.T) {
return res
}
res := mustQuery(t, "ingest", "Row(set=0)")
res := mustQuery(t, c.Idx(), "Row(set=0)")
r := res.Results[0].(*pilosa.Row).Columns()
if len(r) != 1 || r[0] != pilosa.ShardWidth*8+7 {
t.Fatalf("expected row with pilosa.ShardWidth*8+7 set, got %d", r)
}
res = mustQuery(t, "ingest", "Row(set=1)")
res = mustQuery(t, c.Idx(), "Row(set=1)")
r = res.Results[0].(*pilosa.Row).Columns()
if len(r) != 1 || r[0] != pilosa.ShardWidth*8+7 {
t.Fatalf("expected row with pilosa.ShardWidth*8+7 set, got %d", r)
}
res = mustQuery(t, "ingest", "Row(int==1)")
res = mustQuery(t, c.Idx(), "Row(int==1)")
r = res.Results[0].(*pilosa.Row).Columns()
if len(r) != 1 || r[0] != pilosa.ShardWidth*8+7 {
t.Fatalf("expected row with, pilosa.ShardWidth*8+7 set, got %d", r)
@ -699,23 +655,23 @@ func TestAPI_Ingest(t *testing.T) {
},
},
}
if err := coord.API.ImportRoaringShard(context.Background(), "ingest", 8, request); err != nil {
if err := coord.API.ImportRoaringShard(context.Background(), c.Idx(), 8, request); err != nil {
t.Fatalf("ingesting: %v", err)
}
res = mustQuery(t, "ingest", "Row(set=0)")
res = mustQuery(t, c.Idx(), "Row(set=0)")
r = res.Results[0].(*pilosa.Row).Columns()
if len(r) != 0 {
t.Fatalf("expected no values after clearing, got: %v", r)
}
res = mustQuery(t, "ingest", "Row(set=1)")
res = mustQuery(t, c.Idx(), "Row(set=1)")
r = res.Results[0].(*pilosa.Row).Columns()
if len(r) != 0 {
t.Fatalf("expected no values after clearing, got: %v", r)
}
res = mustQuery(t, "ingest", "Row(int==1)")
res = mustQuery(t, c.Idx(), "Row(int==1)")
r = res.Results[0].(*pilosa.Row).Columns()
if len(r) != 0 {
t.Fatalf("expected no values after clearing, got: %v", r)
@ -747,14 +703,7 @@ func BenchmarkIngest(b *testing.B) {
data := ingestBenchmarkHelper()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := test.MustRunCluster(b, 1,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(b, 1)
defer c.Close()
coord := c.GetPrimary()
@ -762,7 +711,7 @@ func BenchmarkIngest(b *testing.B) {
// m1 := c.GetNode(1)
// m2 := c.GetNode(2)
index := "ingest"
index := c.Idx()
setField := "set"
intField := "int"
tqField := "tq"
@ -798,24 +747,8 @@ func BenchmarkIngest(b *testing.B) {
}
}
// offsetModHasher represents a simple, mod-based hashing offset by 1.
type offsetModHasher struct{}
func (*offsetModHasher) Hash(key uint64, n int) int {
return int(key+1) % n
}
func (*offsetModHasher) Name() string { return "mod" }
func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
c := test.MustRunCluster(t, 1,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(t, 1)
defer c.Close()
// plan:
@ -828,7 +761,7 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
m0api := m0.API
ctx := context.Background()
index := "i"
index := c.Idx()
fieldAcct0 := "acct0"
opts := pilosa.OptFieldTypeInt(-1000, 1000)
@ -1072,7 +1005,9 @@ type mutexCheckField struct {
}
func TestAPI_MutexCheck(t *testing.T) {
c := test.MustNewCluster(t, 3)
// Can't share this one, because it has to get custom option settings and needs
// replication.
c := test.MustUnsharedCluster(t, 3)
for _, c := range c.Nodes {
c.Config.Cluster.ReplicaN = 2
}
@ -1094,7 +1029,7 @@ func TestAPI_MutexCheck(t *testing.T) {
ctx := context.Background()
for _, keyedIndex := range []bool{false, true} {
indexName := fmt.Sprintf("i%t", keyedIndex)
indexName := c.Idx(map[bool]string{false: "u", true: "k"}[keyedIndex])
index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: keyedIndex, TrackExistence: true})
if err != nil {
t.Fatalf("creating index: %v", err)
@ -1446,14 +1381,14 @@ func createFieldForTest(index string, field string, coord *test.Command, t *test
func TestVariousApiTranslateCalls(t *testing.T) {
for i := 1; i < 8; i += 3 {
m := test.MustRunCluster(t, i)
defer m.Close()
node := m.GetNode(0)
c := test.MustRunCluster(t, i)
defer c.Close()
node := c.GetNode(0)
api := node.API
// this should never actually get used because we're testing for errors here
r := strings.NewReader("")
// test index
idx, err := api.Holder().CreateIndex("index", pilosa.IndexOptions{})
idx, err := api.Holder().CreateIndex(c.Idx(), pilosa.IndexOptions{})
if err != nil {
t.Fatalf("%v: could not create test index", err)
}
@ -1471,8 +1406,8 @@ func TestVariousApiTranslateCalls(t *testing.T) {
t.Run("translateIndexDbOnNilTranslateStore",
func(t *testing.T) {
err := api.TranslateIndexDB(context.Background(), "index", 0, r)
expected := fmt.Errorf("index %q has no translate store", "index")
err := api.TranslateIndexDB(context.Background(), c.Idx(), 0, r)
expected := fmt.Errorf("index %q has no translate store", c.Idx())
if !reflect.DeepEqual(err, expected) {
t.Fatalf("expected '%#v', got '%#v'", expected, err)
}
@ -1489,8 +1424,8 @@ func TestVariousApiTranslateCalls(t *testing.T) {
t.Run("translateFieldDbOnNilField",
func(t *testing.T) {
err := api.TranslateFieldDB(context.Background(), "index", "nonExistentField", r)
expected := fmt.Errorf("field %q/%q not found", "index", "nonExistentField")
err := api.TranslateFieldDB(context.Background(), c.Idx(), "nonExistentField", r)
expected := fmt.Errorf("field %q/%q not found", c.Idx(), "nonExistentField")
if !reflect.DeepEqual(err, expected) {
t.Fatalf("expected '%#v', got '%#v'", expected, err)
}
@ -1498,7 +1433,7 @@ func TestVariousApiTranslateCalls(t *testing.T) {
t.Run("translateFieldDbNilField_keys",
func(t *testing.T) {
err := api.TranslateFieldDB(context.Background(), "index", "_keys", r)
err := api.TranslateFieldDB(context.Background(), c.Idx(), "_keys", r)
if err != nil {
t.Fatalf("expected 'nil', got '%#v'", err)
}
@ -1508,8 +1443,8 @@ func TestVariousApiTranslateCalls(t *testing.T) {
stores, which is a bug, but one that we will eventually fix. when we do, this
test might come in handy t.Run("translateFieldDbOnNilTranslateStore",
func(t *testing.T) {
err := api.TranslateFieldDB(context.Background(), "index", "field", r)
expected := fmt.Errorf("field %q/%q has no translate store", "index", "field")
err := api.TranslateFieldDB(context.Background(), c.Idx(), "field", r)
expected := fmt.Errorf("field %q/%q has no translate store", c.Idx(), "field")
if !reflect.DeepEqual(err, expected) {
t.Fatalf("expected '%#v', got '%#v'", expected, err)
}
@ -1529,7 +1464,7 @@ func TestAPI_CreateField(t *testing.T) {
nodes[i] = c.GetNode(i)
}
if _, err := nodes[0].API.CreateIndex(ctx, "i", pilosa.IndexOptions{}); err != nil {
if _, err := nodes[0].API.CreateIndex(ctx, c.Idx(), pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
eg, ctx := errgroup.WithContext(context.Background())
@ -1537,7 +1472,7 @@ func TestAPI_CreateField(t *testing.T) {
node := n
eg.Go(func() error {
for i := 0; i < 10; i++ {
_, err := node.API.CreateField(ctx, "i", fmt.Sprintf("f%d", i))
_, err := node.API.CreateField(ctx, c.Idx(), fmt.Sprintf("f%d", i))
if err != nil && !errors.Is(err, pilosa.ErrFieldExists) {
return err
}
@ -1557,19 +1492,12 @@ func TestAPI_CreateField(t *testing.T) {
func TestAPI_RBFDebugInfo(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := test.MustRunCluster(t, 1,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(t, 1)
defer c.Close()
coord := c.GetPrimary()
if _, err := coord.API.CreateIndex(ctx, "i", pilosa.IndexOptions{}); err != nil {
if _, err := coord.API.CreateIndex(ctx, c.Idx(), pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if infos := coord.API.RBFDebugInfo(); infos == nil {
t.Fatal("expected info")
@ -1699,25 +1627,22 @@ f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA==
config.TLS.CertificateKeyPath = writeTestFile(t, "certKey.pem", localhostKey)
config.TLS.CertificatePath = writeTestFile(t, "cert.pem", localhostCert)
c := test.MustRunCluster(t, 3,
c := test.MustRunUnsharedCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&test.ModHasher{}),
),
server.OptCommandConfig(config),
},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerClusterHasher(&test.ModHasher{}),
),
server.OptCommandConfig(config),
},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerClusterHasher(&test.ModHasher{}),
),
server.OptCommandConfig(config),
},
@ -1727,6 +1652,8 @@ f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA==
primaryAPI := c.GetPrimary().API
// needs internal/cluster/message
// Note: This indexName wouldn't be safe on a shared cluster, but we have to use an
// unshared cluster to set up the auth config anyway.
indexName := "test"
_, err := primaryAPI.CreateIndex(adminCtx, indexName, pilosa.IndexOptions{})
if err != nil {

2
client/client_it_test.go
View file

@ -71,7 +71,7 @@ func TestClientAgainstCluster(t *testing.T) {
t.Run(testName, func(t *testing.T) {
// Start size.replicaN cluster
c := test.MustNewCluster(t, size)
c := test.MustUnsharedCluster(t, size)
for _, n := range c.Nodes {
n.Config.Cluster.ReplicaN = replicaN
}

50
cluster_internal_test.go
View file

@ -10,11 +10,9 @@ import (
"testing/quick"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/featurebasedb/featurebase/v3/disco"
pnet "github.com/featurebasedb/featurebase/v3/net"
"github.com/featurebasedb/featurebase/v3/roaring"
_ "github.com/featurebasedb/featurebase/v3/vprint"
"github.com/molecula/featurebase/v3/disco"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/roaring"
)
// Ensure the cluster can fairly distribute partitions across the nodes.
@ -25,7 +23,7 @@ func TestCluster_Owners(t *testing.T) {
{URI: NewTestURIFromHostPort("serverB", 1000)},
{URI: NewTestURIFromHostPort("serverC", 1000)},
}),
Hasher: NewTestModHasher(),
Hasher: &disco.Jmphasher{},
ReplicaN: 2,
}
@ -34,14 +32,24 @@ func TestCluster_Owners(t *testing.T) {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := c.NewSnapshot()
// Verify nodes are distributed.
if a := snap.PartitionNodes(0); !reflect.DeepEqual(a, []*disco.Node{cNodes[0], cNodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
assigned := make(map[int]int)
for i := 0; i < 256; i++ {
nodes := snap.PartitionNodes(i)
for _, node := range nodes {
for j, n := range cNodes {
if n == node {
assigned[j]++
}
}
}
}
// Verify nodes go around the ring.
if a := snap.PartitionNodes(2); !reflect.DeepEqual(a, []*disco.Node{cNodes[2], cNodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
expected := float64((256.0 * 2) / 3) // each partition is on two nodes, there's three nodes
for k, v := range assigned {
ratio := float64(v) / expected
// Empirically, we expect 167/171/174
if ratio < 0.97 || ratio > 1.03 {
t.Fatalf("node %d has %d assigned partitions, expected about %.1f", k, v, expected)
}
}
}
@ -98,10 +106,18 @@ func TestCluster_ContainsShards(t *testing.T) {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := c.NewSnapshot()
shards := snap.ContainsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), cNodes[2])
if !reflect.DeepEqual(shards, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected shars for node's index: %v", shards)
availableShards := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
nodeCounts := make(map[uint64]int)
for _, n := range cNodes {
shards := snap.ContainsShards("test", availableShards, n)
for _, shard := range shards {
nodeCounts[shard]++
}
}
for shard, count := range nodeCounts {
if count != 3 {
t.Fatalf("shard %d on %d nodes, expected 3", shard, count)
}
}
}

33
ctl/import_test.go
View file

@ -217,18 +217,18 @@ func TestImportCommand_RunKeys(t *testing.T) {
cmd := cluster.GetNode(0)
cm.Host = cmd.API.Node().URI.HostPort()
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", fmt.Sprintf("http://%s/index/%i", cm.Host, cluster), strings.NewReader(`{"options":{"keys": true}}`)))
if err != nil {
t.Fatalf("posting request: %v", err)
}
resp.Body.Close()
resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`)))
resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", fmt.Sprintf("http://%s/index/%i/field/f", cm.Host, cluster), strings.NewReader(`{"options":{"keys": true}}`)))
if err != nil {
t.Fatalf("posting request: %v", err)
}
resp.Body.Close()
cm.Index = "i"
cm.Index = cluster.Idx("i")
cm.Field = "f"
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
@ -272,18 +272,18 @@ func TestImportCommand_KeyReplication(t *testing.T) {
cm.Host = host0
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", fmt.Sprintf("http://%s/index/%i", cm.Host, c), strings.NewReader(`{"options":{"keys": true}}`)))
if err != nil {
t.Fatalf("posting request: %v", err)
}
resp.Body.Close()
resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`)))
resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", fmt.Sprintf("http://%s/index/%i/field/f", cm.Host, c), strings.NewReader(`{"options":{"keys": true}}`)))
if err != nil {
t.Fatalf("posting request: %v", err)
}
resp.Body.Close()
cm.Index = "i"
cm.Index = c.Idx("i")
cm.Field = "f"
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
@ -295,7 +295,7 @@ func TestImportCommand_KeyReplication(t *testing.T) {
for _, host := range []string{host0, host1} {
if err := test.RetryUntil(2*time.Second, func() error {
qry := "Count(Row(f=foo0))"
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", fmt.Sprintf("http://%s/index/%i/query", host, c), strings.NewReader(qry)))
if err != nil {
return fmt.Errorf("Querying data for validation: %s", err)
}
@ -335,18 +335,18 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
cmd := cluster.GetNode(0)
cm.Host = cmd.API.Node().URI.HostPort()
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", fmt.Sprintf("http://%s/index/%i", cm.Host, cluster), strings.NewReader(`{"options":{"keys": true}}`)))
if err != nil {
t.Fatalf("posting request: %v", err)
}
resp.Body.Close()
resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`)))
resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", fmt.Sprintf("http://%s/index/%i/field/f", cm.Host, cluster), strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`)))
if err != nil {
t.Fatalf("posting request: %v", err)
}
resp.Body.Close()
cm.Index = "i"
cm.Index = cluster.Idx("i")
cm.Field = "f"
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
@ -463,18 +463,18 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) {
cm.Host = cmd.API.Node().URI.HostPort()
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", fmt.Sprintf("http://%s/index/%i", cm.Host, cluster), strings.NewReader("")))
if err != nil {
t.Fatalf("posting request: %v", err)
}
resp.Body.Close()
resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`)))
resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", fmt.Sprintf("http://%s/index/%i/field/f", cm.Host, cluster), strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`)))
if err != nil {
t.Fatalf("posting request: %v", err)
}
resp.Body.Close()
cm.Index = "i"
cm.Index = cluster.Idx("i")
cm.Field = "f"
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
@ -525,18 +525,18 @@ func TestImportCommand_RunBool(t *testing.T) {
cmd := cluster.GetNode(0)
cm.Host = cmd.API.Node().URI.HostPort()
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", fmt.Sprintf("http://%s/index/%i", cm.Host, cluster), strings.NewReader("")))
if err != nil {
t.Fatalf("posting request: %v", err)
}
resp.Body.Close()
resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "bool"}}`)))
resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", fmt.Sprintf("http://%s/index/%i/field/f", cm.Host, cluster), strings.NewReader(`{"options":{"type": "bool"}}`)))
if err != nil {
t.Fatalf("posting request: %v", err)
}
resp.Body.Close()
cm.Index = "i"
cm.Index = cluster.Idx("i")
cm.Field = "f"
t.Run("Valid", func(t *testing.T) {
@ -711,6 +711,7 @@ func TestImport_AuthOn(t *testing.T) {
t.Fatal(err)
}
// note: cluster isn't shared because of custom opts
cluster := test.MustRunCluster(t, clusterSize, commandOpts...)
defer cluster.Close()
cmd := cluster.GetNode(0)

30
dbshard_test.go
View file

@ -5,34 +5,23 @@ package pilosa_test
import (
"context"
"fmt"
"reflect"
"testing"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/boltdb"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/test"
. "github.com/featurebasedb/featurebase/v3/vprint" // nolint:staticcheck
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/test"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
)
func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
c := test.MustRunCluster(t, 1,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(t, 1)
defer c.Close()
m0 := c.GetNode(0)
t.Run("RowIDColumnKey", func(t *testing.T) {
ctx := context.Background()
indexName := "rick"
indexName := c.Idx()
fieldName := "f"
index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
@ -66,7 +55,7 @@ func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
// Import data with keys to the primary and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
// translated and forwarded to the owner of shard 0
req := &pilosa.ImportRequest{
Index: indexName,
IndexCreatedAt: index.CreatedAt(),
@ -89,9 +78,12 @@ func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
if err != nil {
t.Fatal(err)
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
}
keys := res.Results[0].(*pilosa.Row).Keys
if !sameStringSlice(keys, colKeys) {
t.Fatalf("unexpected column keys: %#v", keys)
}
})

27
delete_test.go
View file

@ -6,6 +6,7 @@ import (
"context"
"fmt"
"math"
"sort"
"testing"
"time"
@ -16,7 +17,9 @@ import (
)
func TestExecutor_DeleteRecords(t *testing.T) {
indexName := "i"
c := test.MustRunCluster(t, 1)
indexName := c.Idx()
defer c.Close()
setup := func(t *testing.T, r *require.Assertions, c *test.Cluster) {
t.Helper()
c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, "setfield")
@ -111,13 +114,6 @@ func TestExecutor_DeleteRecords(t *testing.T) {
}
require := require.New(t)
t.Run("DeleteRecords", func(t *testing.T) {
c := test.MustNewCluster(t, 1)
for _, n := range c.Nodes {
n.Config.Cluster.ReplicaN = 1
}
err := c.Start()
require.NoError(err, "Start cluster DeleteRecords")
defer c.Close()
t.Run("Delete", func(t *testing.T) {
setup(t, require, c)
@ -144,7 +140,9 @@ func TestExecutor_DeleteRecords(t *testing.T) {
resp := c.Query(t, indexName, `Extract(All())`)
m := resp.Results[0].(pilosa.ExtractedTable)
before := convertKey(m.Columns)
require.Equal([]string{"one", "A", "B", "C", "D", "two"}, before, "these keyed records before")
sort.Strings(before)
expected := []string{"A", "B", "C", "D", "one", "two"}
require.Equal(expected, before, "these keyed records before")
resp = c.Query(t, indexName, `Delete(ConstRow(columns=["A","one"]))`)
require.NotEmpty(resp.Results)
require.Equal(true, resp.Results[0], "Change should have happened")
@ -152,6 +150,7 @@ func TestExecutor_DeleteRecords(t *testing.T) {
resp = c.Query(t, indexName, `Extract(All())`)
m = resp.Results[0].(pilosa.ExtractedTable)
after := convertKey(m.Columns)
sort.Strings(after)
require.Equal([]string{"B", "C", "D", "two"}, after, "these keyed records after delete")
//validate that column keys got deleted
node := c.GetNode(0)
@ -240,13 +239,9 @@ func TestExecutor_DeleteRecords(t *testing.T) {
})
})
t.Run("DeleteRecordsBigWithRestart", func(t *testing.T) {
c := test.MustNewCluster(t, 1)
for _, n := range c.Nodes {
n.Config.Cluster.ReplicaN = 1
}
err := c.Start()
// restarting doesn't work correctly for a shared cluster
c := test.MustRunUnsharedCluster(t, 1)
defer c.Close()
require.NoError(err, "Start cluster DeleteRecordsBig")
setupBig(t, require, c, 16)
defer tearDown(t, require, c)
node := c.GetNode(0)
@ -258,7 +253,7 @@ func TestExecutor_DeleteRecords(t *testing.T) {
require.NotNil(resp, "Response should not be nil")
require.NotEmpty(resp.Results)
require.Equal(uint64(0), resp.Results[0], "Should have removed")
err = node.Reopen()
err := node.Reopen()
require.NoError(err, "restart cluster DeleteRecordsBig")
err = c.AwaitState(disco.ClusterStateNormal, 10*time.Second)
require.NoError(err, "backToNormal")

18
executor.go
View file

@ -3052,7 +3052,23 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c
if err := ctx.Err(); err != nil {
return err
}
return mergeGroupCounts(other, findGroupCounts(v), limit)
x := mergeGroupCounts(other, findGroupCounts(v), limit)
for i := range x {
gc := &x[i]
for j := range gc.Group {
fr := &gc.Group[j]
if fr.FieldOptions == nil {
// oops, options were omitted possibly by a remote. try to
// guess them from our local options
field := e.Holder.Field(index, fr.Field)
if field != nil {
options := field.Options()
fr.FieldOptions = &options
}
}
}
}
return x
}
// Get full result set.
other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)

25
executor_internal_test.go
View file

@ -7,6 +7,7 @@ import (
"encoding/json"
"fmt"
"reflect"
"sort"
"strconv"
"testing"
"time"
@ -16,6 +17,30 @@ import (
"github.com/stretchr/testify/assert"
)
// AssertEqual checks a given RowIdentifiers against expected values.
func (r *RowIdentifiers) AssertEqual(tb testing.TB, other *RowIdentifiers) {
sort.Slice(r.Rows, func(i, j int) bool { return r.Rows[i] < r.Rows[j] })
sort.Slice(other.Rows, func(i, j int) bool { return other.Rows[i] < other.Rows[j] })
if len(r.Rows) != len(other.Rows) {
tb.Fatalf("row ID mismatch: got %d, expected %d", r.Rows, other.Rows)
}
for i := range r.Rows {
if r.Rows[i] != other.Rows[i] {
tb.Fatalf("row ID mismatch: got %d, expected %d", r.Rows, other.Rows)
}
}
sort.Strings(r.Keys)
sort.Strings(other.Keys)
if len(r.Keys) != len(other.Keys) {
tb.Fatalf("row keys mismatch: got %s, expected %s", r.Keys, other.Keys)
}
for i := range r.Keys {
if r.Keys[i] != other.Keys[i] {
tb.Fatalf("row keys mismatch: got %s, expected %s", r.Keys, other.Keys)
}
}
}
func TestExecutor_TranslateRowsOnBool(t *testing.T) {
path, _ := testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
holder := NewHolder(path, mustHolderConfig())

1531
executor_test.go
View file

File diff suppressed because it is too large Load diff

154
holder_test.go
View file

@ -218,7 +218,7 @@ func TestHolder_DeleteIndex(t *testing.T) {
// Ensure holder can sync with a remote holder.
func TestHolderSyncer_SyncHolder(t *testing.T) {
c := test.MustNewCluster(t, 2)
c := test.MustUnsharedCluster(t, 2)
c.GetIdleNode(0).Config.Cluster.ReplicaN = 2
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 2
@ -230,27 +230,27 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
}
defer c.Close()
_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_, err = c.GetNode(0).API.CreateIndex(context.Background(), c.Idx(), pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index i: %v", err)
}
_, err = c.GetNode(0).API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{})
_, err = c.GetNode(0).API.CreateIndex(context.Background(), c.Idx("y"), pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index y: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
_, err = c.GetNode(0).API.CreateField(context.Background(), c.Idx(), "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
if err != nil {
t.Fatalf("creating field f: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
_, err = c.GetNode(0).API.CreateField(context.Background(), c.Idx(), "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
if err != nil {
t.Fatalf("creating field f0: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
_, err = c.GetNode(0).API.CreateField(context.Background(), c.Idx("y"), "z", pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
if err != nil {
t.Fatalf("creating field z in y: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "y", "b", pilosa.OptFieldTypeBool())
_, err = c.GetNode(0).API.CreateField(context.Background(), c.Idx("y"), "b", pilosa.OptFieldTypeBool())
if err != nil {
t.Fatalf("creating field b in y: %v", err)
}
@ -259,29 +259,29 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
// Set data on the local holder.
hldr0.SetBit("i", "f", 0, 10)
hldr0.SetBit("i", "f", 2, 20)
hldr0.SetBit("i", "f", 120, 10)
hldr0.SetBit("i", "f", 200, 4)
hldr0.SetBit(c.Idx(), "f", 0, 10)
hldr0.SetBit(c.Idx(), "f", 2, 20)
hldr0.SetBit(c.Idx(), "f", 120, 10)
hldr0.SetBit(c.Idx(), "f", 200, 4)
hldr0.SetBit("i", "f0", 9, ShardWidth+5)
hldr0.SetBit(c.Idx(), "f0", 9, ShardWidth+5)
// Set a bit to create the fragment.
hldr0.SetBit("y", "z", 0, 0)
hldr0.SetBit("y", "b", 0, 0) // rowID = 0 means false
hldr0.SetBit(c.Idx("y"), "z", 0, 0)
hldr0.SetBit(c.Idx("y"), "b", 0, 0) // rowID = 0 means false
// Set data on the remote holder.
hldr1.SetBit("i", "f", 0, 4000)
hldr1.SetBit("i", "f", 3, 10)
hldr1.SetBit("i", "f", 120, 10)
hldr1.SetBit(c.Idx(), "f", 0, 4000)
hldr1.SetBit(c.Idx(), "f", 3, 10)
hldr1.SetBit(c.Idx(), "f", 120, 10)
hldr1.SetBit("y", "z", 10, (3*ShardWidth)+4)
hldr1.SetBit("y", "z", 10, (3*ShardWidth)+5)
hldr1.SetBit("y", "z", 10, (3*ShardWidth)+7)
hldr1.SetBit(c.Idx("y"), "z", 10, (3*ShardWidth)+4)
hldr1.SetBit(c.Idx("y"), "z", 10, (3*ShardWidth)+5)
hldr1.SetBit(c.Idx("y"), "z", 10, (3*ShardWidth)+7)
hldr1.SetBit("y", "b", 1, (3*ShardWidth)+4) // true
hldr1.SetBit("y", "b", 0, (3*ShardWidth)+5) // false
hldr1.SetBit("y", "b", 1, (3*ShardWidth)+7) // true
hldr1.SetBit(c.Idx("y"), "b", 1, (3*ShardWidth)+4) // true
hldr1.SetBit(c.Idx("y"), "b", 0, (3*ShardWidth)+5) // false
hldr1.SetBit(c.Idx("y"), "b", 1, (3*ShardWidth)+7) // true
err = c.GetNode(0).Server.SyncData()
if err != nil {
@ -294,34 +294,34 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
if a := hldr.Row(c.Idx(), "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Errorf("unexpected columns(%d/0): %+v", i, a)
}
if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
if a := hldr.Row(c.Idx(), "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Errorf("unexpected columns(%d/2): %+v", i, a)
}
if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
if a := hldr.Row(c.Idx(), "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Errorf("unexpected columns(%d/3): %+v", i, a)
}
if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
if a := hldr.Row(c.Idx(), "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Errorf("unexpected columns(%d/120): %+v", i, a)
}
if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
if a := hldr.Row(c.Idx(), "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Errorf("unexpected columns(%d/200): %+v", i, a)
}
if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) {
if a := hldr.Row(c.Idx(), "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) {
t.Errorf("unexpected columns(%d/d/f0): %+v", i, a)
}
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) {
if a := hldr.Row(c.Idx("y"), "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) {
t.Errorf("unexpected columns(%d/y/z): %+v", i, a)
}
if a := hldr.Row("y", "b", 0).Columns(); !reflect.DeepEqual(a, []uint64{0, (3 * ShardWidth) + 5}) {
if a := hldr.Row(c.Idx("y"), "b", 0).Columns(); !reflect.DeepEqual(a, []uint64{0, (3 * ShardWidth) + 5}) {
t.Errorf("unexpected false columns(%d/y/b): %+v", i, a)
}
if a := hldr.Row("y", "b", 1).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 7}) {
if a := hldr.Row(c.Idx("y"), "b", 1).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 7}) {
t.Errorf("unexpected true columns(%d/y/b): %+v", i, a)
}
}
@ -330,7 +330,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Ensure holder can sync with a remote holder and respects
// the row boundaries of the block.
func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
c := test.MustNewCluster(t, 3)
c := test.MustUnsharedCluster(t, 3)
c.GetIdleNode(0).Config.Cluster.ReplicaN = 3
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 3
@ -343,11 +343,11 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
}
defer c.Close()
_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_, err = c.GetNode(0).API.CreateIndex(context.Background(), c.Idx(), pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index i: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
_, err = c.GetNode(0).API.CreateField(context.Background(), c.Idx(), "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
if err != nil {
t.Fatalf("creating field f: %v", err)
}
@ -359,13 +359,13 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
hldr2 := &test.Holder{Holder: c.GetNode(2).Server.Holder()}
// Set data on the local holder.
hldr0.SetBit("i", "f", blockEdge-1, 10)
hldr0.SetBit("i", "f", blockEdge, 20)
hldr0.SetBit(c.Idx(), "f", blockEdge-1, 10)
hldr0.SetBit(c.Idx(), "f", blockEdge, 20)
// Set the same data on one of the replicas
// so that we have a quorum.
hldr1.SetBit("i", "f", blockEdge-1, 10)
hldr1.SetBit("i", "f", blockEdge, 20)
hldr1.SetBit(c.Idx(), "f", blockEdge-1, 10)
hldr1.SetBit(c.Idx(), "f", blockEdge, 20)
// Leave the third replica empty to force a block merge.
//
@ -376,10 +376,10 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
// Verify data is the same on all nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1, hldr2} {
if a := hldr.Row("i", "f", blockEdge-1).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
if a := hldr.Row(c.Idx(), "f", blockEdge-1).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Errorf("unexpected columns(%d/block 0): %+v", i, a)
}
if a := hldr.Row("i", "f", blockEdge).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
if a := hldr.Row(c.Idx(), "f", blockEdge).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Errorf("unexpected columns(%d/block 1): %+v", i, a)
}
}
@ -387,7 +387,7 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
// Ensure holder correctly handles clears during block sync.
func TestHolderSyncer_Clears(t *testing.T) {
c := test.MustNewCluster(t, 3)
c := test.MustUnsharedCluster(t, 3)
c.GetIdleNode(0).Config.Cluster.ReplicaN = 3
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 3
@ -400,11 +400,11 @@ func TestHolderSyncer_Clears(t *testing.T) {
}
defer c.Close()
_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_, err = c.GetNode(0).API.CreateIndex(context.Background(), c.Idx(), pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index i: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
_, err = c.GetNode(0).API.CreateField(context.Background(), c.Idx(), "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
if err != nil {
t.Fatalf("creating field f: %v", err)
}
@ -415,13 +415,13 @@ func TestHolderSyncer_Clears(t *testing.T) {
// Set data on the local holder that should be cleared
// because it's the only instance of this value.
hldr0.SetBit("i", "f", 0, 30)
hldr0.SetBit(c.Idx(), "f", 0, 30)
// Set similar data on the replicas, but
// different from what's on local. This should end
// up being set on all replicas
hldr1.SetBit("i", "f", 0, 20)
hldr2.SetBit("i", "f", 0, 20)
hldr1.SetBit(c.Idx(), "f", 0, 20)
hldr2.SetBit(c.Idx(), "f", 0, 20)
err = c.GetNode(0).Server.SyncData()
if err != nil {
@ -430,7 +430,7 @@ func TestHolderSyncer_Clears(t *testing.T) {
// Verify data is the same on all nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1, hldr2} {
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
if a := hldr.Row(c.Idx(), "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Errorf("unexpected columns(%d): %+v", i, a)
}
}
@ -438,7 +438,7 @@ func TestHolderSyncer_Clears(t *testing.T) {
// Ensure holder can sync time quantum views with a remote holder.
func TestHolderSyncer_TimeQuantum(t *testing.T) {
c := test.MustNewCluster(t, 2)
c := test.MustUnsharedCluster(t, 2)
c.GetIdleNode(0).Config.Cluster.ReplicaN = 2
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 2
@ -451,11 +451,11 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
quantum := "D"
_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_, err = c.GetNode(0).API.CreateIndex(context.Background(), c.Idx(), pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index i: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum), "0"))
_, err = c.GetNode(0).API.CreateField(context.Background(), c.Idx(), "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum), "0"))
if err != nil {
t.Fatalf("creating field f: %v", err)
}
@ -466,11 +466,11 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
// Set data on the local holder for node0.
t1 := time.Date(2018, 8, 1, 12, 30, 0, 0, time.UTC)
t2 := time.Date(2018, 8, 2, 12, 30, 0, 0, time.UTC)
hldr0.SetBitTime("i", "f", 0, 1, &t1)
hldr0.SetBitTime("i", "f", 0, 2, &t2)
hldr0.SetBitTime(c.Idx(), "f", 0, 1, &t1)
hldr0.SetBitTime(c.Idx(), "f", 0, 2, &t2)
// Set data on node1.
hldr1.SetBitTime("i", "f", 0, 22, &t2)
hldr1.SetBitTime(c.Idx(), "f", 0, 22, &t2)
err = c.GetNode(0).Server.SyncData()
if err != nil {
@ -479,10 +479,10 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
if a := hldr.RowTime("i", "f", 0, t1, quantum).Columns(); !reflect.DeepEqual(a, []uint64{1}) {
if a := hldr.RowTime(c.Idx(), "f", 0, t1, quantum).Columns(); !reflect.DeepEqual(a, []uint64{1}) {
t.Errorf("unexpected columns(%d/0): %+v", i, a)
}
if a := hldr.RowTime("i", "f", 0, t2, quantum).Columns(); !reflect.DeepEqual(a, []uint64{2, 22}) {
if a := hldr.RowTime(c.Idx(), "f", 0, t2, quantum).Columns(); !reflect.DeepEqual(a, []uint64{2, 22}) {
t.Errorf("unexpected columns(%d/0): %+v", i, a)
}
}
@ -491,7 +491,7 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
// Ensure holder can sync integer views with a remote holder.
func TestHolderSyncer_IntField(t *testing.T) {
t.Run("BasicSync", func(t *testing.T) {
c := test.MustNewCluster(t, 2)
c := test.MustUnsharedCluster(t, 2)
c.GetIdleNode(0).Config.Cluster.ReplicaN = 2
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 2
@ -503,12 +503,12 @@ func TestHolderSyncer_IntField(t *testing.T) {
defer c.Close()
var idx0 *pilosa.Index
idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), c.Idx(), pilosa.IndexOptions{})
_ = idx0
if err != nil {
t.Fatalf("creating index i: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(0, 100))
_, err = c.GetNode(0).API.CreateField(context.Background(), c.Idx(), "f", pilosa.OptFieldTypeInt(0, 100))
if err != nil {
t.Fatalf("creating field f: %v", err)
}
@ -517,12 +517,12 @@ func TestHolderSyncer_IntField(t *testing.T) {
hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
// Set data on the local holder for node0. columnID=1, value=1
hldr0.SetValue("i", "f", 1, 1)
hldr0.SetValue(c.Idx(), "f", 1, 1)
// in c0 expect the 1 bit
// Set data on node1. columnID=2, value=2
idx1 := hldr1.SetValue("i", "f", 2, 2)
idx1 := hldr1.SetValue(c.Idx(), "f", 2, 2)
_ = idx1
err = c.GetNode(0).Server.SyncData()
@ -537,11 +537,11 @@ func TestHolderSyncer_IntField(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
if a, exists := hldr.Value("i", "f", 1); !exists || a != 1 {
if a, exists := hldr.Value(c.Idx(), "f", 1); !exists || a != 1 {
// expects exists==true, a==1
t.Errorf("unexpected value(node%d/0): a:%d, exists: %v", i, a, exists)
}
if a, exists := hldr.Value("i", "f", 2); exists {
if a, exists := hldr.Value(c.Idx(), "f", 2); exists {
t.Errorf("unexpected value(node%d/1): a:%d, exists: %v", i, a, exists)
}
}
@ -549,7 +549,7 @@ func TestHolderSyncer_IntField(t *testing.T) {
t.Run("MultiShard", func(t *testing.T) {
t.Skip() // skipping due to changed partitioning strategy
c := test.MustNewCluster(t, 2)
c := test.MustUnsharedCluster(t, 2)
c.GetIdleNode(0).Config.Cluster.ReplicaN = 2
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 2
@ -562,12 +562,12 @@ func TestHolderSyncer_IntField(t *testing.T) {
var idx0 *pilosa.Index
_ = idx0
idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), c.Idx(), pilosa.IndexOptions{})
_ = idx0
if err != nil {
t.Fatalf("creating index i: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
_, err = c.GetNode(0).API.CreateField(context.Background(), c.Idx(), "f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
if err != nil {
t.Fatalf("creating field f: %v", err)
}
@ -576,18 +576,18 @@ func TestHolderSyncer_IntField(t *testing.T) {
hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
// Set data on the local holder for node0.
hldr0.SetValue("i", "f", 1*pilosa.ShardWidth, 11)
hldr0.SetValue("i", "f", 3*pilosa.ShardWidth, 32)
hldr0.SetValue("i", "f", 4*pilosa.ShardWidth, math.MinInt32)
hldr0.SetValue("i", "f", 7*pilosa.ShardWidth, math.MinInt32)
hldr0.SetValue(c.Idx(), "f", 1*pilosa.ShardWidth, 11)
hldr0.SetValue(c.Idx(), "f", 3*pilosa.ShardWidth, 32)
hldr0.SetValue(c.Idx(), "f", 4*pilosa.ShardWidth, math.MinInt32)
hldr0.SetValue(c.Idx(), "f", 7*pilosa.ShardWidth, math.MinInt32)
// Set data on node1.
hldr1.SetValue("i", "f", 0*pilosa.ShardWidth, 2)
hldr1.SetValue("i", "f", 2*pilosa.ShardWidth, 22)
hldr1.SetValue("i", "f", 4*pilosa.ShardWidth, math.MaxInt32)
hldr1.SetValue("i", "f", 7*pilosa.ShardWidth, math.MaxInt32)
hldr1.SetValue(c.Idx(), "f", 0*pilosa.ShardWidth, 2)
hldr1.SetValue(c.Idx(), "f", 2*pilosa.ShardWidth, 22)
hldr1.SetValue(c.Idx(), "f", 4*pilosa.ShardWidth, math.MaxInt32)
hldr1.SetValue(c.Idx(), "f", 7*pilosa.ShardWidth, math.MaxInt32)
// Primary for shards (for index "i"):
// Primary for shards (for index c.Idx()):
// node0: [0,3,7]
// node1: [1,2,4]
@ -604,10 +604,10 @@ func TestHolderSyncer_IntField(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
if a := hldr.Range("i", "f", pql.GT, 0); !reflect.DeepEqual(a.Columns(), []uint64{2 * pilosa.ShardWidth, 3 * pilosa.ShardWidth, 4 * pilosa.ShardWidth}) {
if a := hldr.Range(c.Idx(), "f", pql.GT, 0); !reflect.DeepEqual(a.Columns(), []uint64{2 * pilosa.ShardWidth, 3 * pilosa.ShardWidth, 4 * pilosa.ShardWidth}) {
t.Errorf("unexpected columns(node%d/0): %d", i, a.Columns())
}
if a := hldr.Range("i", "f", pql.LT, 0); !reflect.DeepEqual(a.Columns(), []uint64{7 * pilosa.ShardWidth}) {
if a := hldr.Range(c.Idx(), "f", pql.LT, 0); !reflect.DeepEqual(a.Columns(), []uint64{7 * pilosa.ShardWidth}) {
t.Errorf("unexpected columns(node%d/0): %d", i, a.Columns())
}
}

127
http_handler_test.go
View file

@ -173,10 +173,11 @@ func TestUpdateFieldTTL(t *testing.T) {
},
}
indexName := c.Idx("s")
for i, test := range tests {
t.Run(test.name, func(t *testing.T) {
c.CreateField(t, "ttltest", pilosa.IndexOptions{}, test.name, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), test.ttl))
nodeURL := c.Nodes[0].URL() + "/index/ttltest/field/" + test.field
c.CreateField(t, indexName, pilosa.IndexOptions{}, test.name, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), test.ttl))
nodeURL := fmt.Sprintf("%s/index/%s/field/%s", c.Nodes[0].URL(), c, test.field)
req, err := gohttp.NewRequest("PATCH", nodeURL, strings.NewReader(test.ttlOption))
if err != nil {
t.Fatal(err)
@ -215,12 +216,21 @@ func TestUpdateFieldTTL(t *testing.T) {
if err != nil {
t.Fatalf("getting schema: %v", err)
}
if ii[0].Fields[i].Name == test.name {
if ii[0].Fields[i].Options.TTL != test.expTTL {
t.Errorf("expected TTL: '%s', got: '%s'", test.expTTL, ii[0].Fields[i].Options.TTL)
for _, idx := range ii {
if idx.Name == indexName {
if len(idx.Fields) <= i {
t.Fatalf("expected %d fields, last %s, got %d fields",
i, test.name, len(idx.Fields))
}
if idx.Fields[i].Name == test.name {
if idx.Fields[i].Options.TTL != test.expTTL {
t.Errorf("expected noStandardView value: '%s', got: '%s'", test.expTTL, idx.Fields[i].Options.TTL)
}
} else {
t.Errorf("unexpected field: '%s', got: '%s'", test.name, idx.Fields[i].Name)
}
break
}
} else {
t.Errorf("unexpected field: '%s', got: '%s'", test.name, ii[0].Fields[i].Name)
}
}
@ -278,11 +288,13 @@ func TestUpdateFieldNoStandardView(t *testing.T) {
expNoStandardView: false,
},
}
// "s" to make it match %s behavior of a cluster
indexName := c.Idx("s")
for i, test := range tests {
t.Run(test.name, func(t *testing.T) {
c.CreateField(t, "ttltest", pilosa.IndexOptions{}, test.name, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0"))
nodeURL := c.Nodes[0].URL() + "/index/ttltest/field/" + test.field
c.CreateField(t, indexName, pilosa.IndexOptions{}, test.name, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0"))
nodeURL := fmt.Sprintf("%s/index/%s/field/%s", c.Nodes[0].URL(), c, test.field)
req, err := gohttp.NewRequest("PATCH", nodeURL, strings.NewReader(test.fieldOption))
if err != nil {
t.Fatal(err)
@ -317,33 +329,34 @@ func TestUpdateFieldNoStandardView(t *testing.T) {
if err != nil {
t.Fatalf("getting schema: %v", err)
}
if ii[0].Fields[i].Name == test.name {
if ii[0].Fields[i].Options.NoStandardView != test.expNoStandardView {
t.Errorf("expected noStandardView value: '%t', got: '%t'", test.expNoStandardView, ii[0].Fields[i].Options.NoStandardView)
for _, idx := range ii {
if idx.Name == indexName {
if len(idx.Fields) <= i {
t.Fatalf("expected %d fields, last %s, got %d fields",
i, test.name, len(idx.Fields))
}
if idx.Fields[i].Name == test.name {
if idx.Fields[i].Options.NoStandardView != test.expNoStandardView {
t.Errorf("expected noStandardView value: '%t', got: '%t'", test.expNoStandardView, idx.Fields[i].Options.NoStandardView)
}
} else {
t.Errorf("unexpected field: '%s', got: '%s'", test.name, idx.Fields[i].Name)
}
break
}
} else {
t.Errorf("unexpected field: '%s', got: '%s'", test.name, ii[0].Fields[i].Name)
}
}
})
}
}
func TestIngestSchemaHandler(t *testing.T) {
c := test.MustRunCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&test.ModHasher{}))},
[]server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&test.ModHasher{}))},
[]server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&test.ModHasher{}))},
)
c := test.MustRunCluster(t, 3)
defer c.Close()
schema := `
schema := fmt.Sprintf(`
{
"index-name": "example",
"index-name": "%s",
"primary-key-type": "string",
"index-action": "create",
"fields": [
@ -408,7 +421,7 @@ func TestIngestSchemaHandler(t *testing.T) {
}
]
}
`
`, c)
m := c.GetPrimary()
schemaURL := fmt.Sprintf("%s/internal/schema", m.URL())
resp := test.Do(t, "POST", schemaURL, string(schema))
@ -425,15 +438,16 @@ func TestIngestSchemaHandler(t *testing.T) {
func TestPostFieldWithTTL(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
indexName := c.Idx("%s")
schema := `
schema := fmt.Sprintf(`
{
"index-name": "ttl_test",
"index-name": "%s",
"primary-key-type": "string",
"index-action": "create",
"fields":[]
}
`
`, c)
m := c.GetPrimary()
schemaURL := fmt.Sprintf("%s/internal/schema", m.URL())
resp := test.Do(t, "POST", schemaURL, string(schema))
@ -451,7 +465,7 @@ func TestPostFieldWithTTL(t *testing.T) {
}{
{
name: "t1_48h",
url: fmt.Sprintf("%s/index/ttl_test/field/t1_48h", m.URL()),
url: fmt.Sprintf("%s/index/%s/field/t1_48h", m.URL(), c),
option: `{ "options": {"timeQuantum":"YMDH","type":"time","ttl":"48h" }}`,
expStatus: 200,
expErr: `"success":true`,
@ -459,28 +473,28 @@ func TestPostFieldWithTTL(t *testing.T) {
},
{
name: "t2_unknown_unit",
url: fmt.Sprintf("%s/index/ttl_test/field/t2_unknown_unit", m.URL()),
url: fmt.Sprintf("%s/index/%s/field/t2_unknown_unit", m.URL(), c),
option: `{ "options": {"timeQuantum":"YMDH","type":"time","ttl":"24abc" }}`,
expStatus: 400,
expErr: "cannot parse ttl",
},
{
name: "t3_invalid",
url: fmt.Sprintf("%s/index/ttl_test/field/t3_invalid", m.URL()),
url: fmt.Sprintf("%s/index/%s/field/t3_invalid", m.URL(), c),
option: `{ "options": {"timeQuantum":"YMDH","type":"time","ttl":"abcdef" }}`,
expStatus: 400,
expErr: "cannot parse ttl",
},
{
name: "t4_invalid_empty",
url: fmt.Sprintf("%s/index/ttl_test/field/t4_invalid_empty", m.URL()),
url: fmt.Sprintf("%s/index/%s/field/t4_invalid_empty", m.URL(), c),
option: `{ "options": {"timeQuantum":"YMDH","type":"time","ttl":"" }}`,
expStatus: 400,
expErr: "cannot parse ttl",
},
{
name: "t5_negative",
url: fmt.Sprintf("%s/index/ttl_test/field/t5_negative", m.URL()),
url: fmt.Sprintf("%s/index/%s/field/t5_negative", m.URL(), c),
option: `{ "options": {"timeQuantum":"YMDH","type":"time","ttl":"-24h" }}`,
expStatus: 400,
expErr: "ttl can't be negative",
@ -508,12 +522,21 @@ func TestPostFieldWithTTL(t *testing.T) {
t.Fatalf("getting schema: %v", err)
}
if ii[0].Fields[i].Name == test_i.name {
if ii[0].Fields[i].Options.TTL != test_i.expTTL {
t.Errorf("expected TTL: '%s', got: '%s'", test_i.expTTL, ii[0].Fields[i].Options.TTL)
for _, idx := range ii {
if idx.Name == indexName {
if len(idx.Fields) <= i {
t.Fatalf("expected %d fields, last %s, got %d fields",
i, test_i.name, len(idx.Fields))
}
if idx.Fields[i].Name == test_i.name {
if idx.Fields[i].Options.TTL != test_i.expTTL {
t.Errorf("expected TTL: '%s', got: '%s'", test_i.expTTL, idx.Fields[i].Options.TTL)
}
} else {
t.Errorf("unexpected field: '%s', got: '%s'", test_i.name, idx.Fields[i].Name)
}
break
}
} else {
t.Errorf("unexpected field: '%s', got: '%s'", test_i.name, ii[0].Fields[i].Name)
}
}
}
@ -525,9 +548,9 @@ func TestGetViewAndDelete(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
schema := `
schema := fmt.Sprintf(`
{
"index-name": "example",
"index-name": "%s",
"primary-key-type": "string",
"index-action": "create",
"fields": [
@ -540,7 +563,7 @@ func TestGetViewAndDelete(t *testing.T) {
}
]
}
`
`, c)
m := c.GetPrimary()
schemaURL := fmt.Sprintf("%s/internal/schema", m.URL())
resp := test.Do(t, "POST", schemaURL, string(schema))
@ -549,7 +572,7 @@ func TestGetViewAndDelete(t *testing.T) {
}
// Send sample data
postQueryUrl := fmt.Sprintf("%s/index/example/query", m.URL())
postQueryUrl := fmt.Sprintf("%s/index/%s/query", m.URL(), c)
queryOption := `
Set(1,test_view=1,2001-02-03T04:05)
`
@ -568,7 +591,7 @@ func TestGetViewAndDelete(t *testing.T) {
}
// Call view to get data
viewUrl := fmt.Sprintf("%s/index/example/field/test_view/view", m.URL())
viewUrl := fmt.Sprintf("%s/index/%s/field/test_view/view", m.URL(), c)
respView := test.Do(t, "GET", viewUrl, "")
if respView.StatusCode != gohttp.StatusOK {
t.Errorf("view handler, status: %d, body=%s", respView.StatusCode, respView.Body)
@ -598,7 +621,7 @@ func TestGetViewAndDelete(t *testing.T) {
}
// call delete on view standard_2001020304
deleteViewUrl := fmt.Sprintf("%s/index/example/field/test_view/view/standard_2001020304", m.URL())
deleteViewUrl := fmt.Sprintf("%s/index/%s/field/test_view/view/standard_2001020304", m.URL(), c)
respDelete := test.Do(t, "DELETE", deleteViewUrl, "")
if respDelete.StatusCode != gohttp.StatusOK {
t.Errorf("delete handler, status: %d, body=%s", respDelete.StatusCode, respDelete.Body)
@ -608,7 +631,7 @@ func TestGetViewAndDelete(t *testing.T) {
expectedViewNames = expectedViewNames[:len(expectedViewNames)-1]
// call view again
viewUrl = fmt.Sprintf("%s/index/example/field/test_view/view", m.URL())
viewUrl = fmt.Sprintf("%s/index/%s/field/test_view/view", m.URL(), c)
respView = test.Do(t, "GET", viewUrl, "")
if respView.StatusCode != gohttp.StatusOK {
t.Errorf("view handler after delete, status: %d, body=%s", respView.StatusCode, respView.Body)
@ -641,9 +664,9 @@ func TestTranslationHandlers(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
schema := `
schema := fmt.Sprintf(`
{
"index-name": "example",
"index-name": "%s",
"primary-key-type": "string",
"index-action": "create",
"fields": [
@ -657,7 +680,7 @@ func TestTranslationHandlers(t *testing.T) {
}
]
}
`
`, c)
m := c.GetPrimary()
schemaURL := fmt.Sprintf("%s/internal/schema", m.URL())
resp := test.Do(t, "POST", schemaURL, string(schema))
@ -665,9 +688,9 @@ func TestTranslationHandlers(t *testing.T) {
t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
baseURLs := []string{
fmt.Sprintf("%s/internal/translate/index/example/", m.URL()),
fmt.Sprintf("%s/internal/translate/field/example/stringset/", m.URL()),
fmt.Sprintf("%s/internal/translate/field/example/nonexistent/", m.URL()),
fmt.Sprintf("%s/internal/translate/index/%s/", m.URL(), c),
fmt.Sprintf("%s/internal/translate/field/%s/stringset/", m.URL(), c),
fmt.Sprintf("%s/internal/translate/field/%s/nonexistent/", m.URL(), c),
}
for _, url := range baseURLs {
expectFailure := strings.HasSuffix(url, "/nonexistent/")

2
index_test.go
View file

@ -261,7 +261,7 @@ func isNotFoundError(err error) bool {
// This is a regression test after a customer experienced the same deadlock.
// For details, check out https://molecula.atlassian.net/browse/CORE-919
func TestIndex_RecreateFieldOnRestart(t *testing.T) {
c := test.MustRunCluster(t, 1)
c := test.MustRunUnsharedCluster(t, 1)
defer c.Close()
// create index

458
internal_client_test.go
View file

@ -15,99 +15,99 @@ import (
"time"
"github.com/davecgh/go-spew/spew"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/authn"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/encoding/proto"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/test"
"github.com/featurebasedb/featurebase/v3/vprint"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/test"
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
"github.com/ricochet2200/go-disk-usage/du"
)
// Test distributed TopN Row count across 3 nodes.
func TestClient_MultiNode(t *testing.T) {
c := test.MustRunCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&test.ModHasher{}))},
[]server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&test.ModHasher{}))},
[]server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&test.ModHasher{}))},
)
c := test.MustRunCluster(t, 3)
defer c.Close()
hldr0 := c.GetHolder(0)
hldr1 := c.GetHolder(1)
hldr2 := c.GetHolder(2)
// Create a dispersed set of bitmaps across 3 nodes such that each
// individual node and shard width increment would reveal a different TopN.
shardNums := []uint64{1, 2, 6}
// This was generated with:
// `owns := s[i].Handler.Handler.API.Cluster.OwnsShards("i", 20, s[i].HostURI())`
owns := [][]uint64{
{1, 3, 4, 8, 10, 13, 17, 19},
{2, 5, 7, 11, 12, 14, 18},
{0, 6, 9, 15, 16, 20},
}
for i, num := range shardNums {
ownsNum := false
for _, ownNum := range owns[i] {
if ownNum == num {
ownsNum = true
break
}
}
if !ownsNum {
t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, c.GetNode(i).URL(), owns)
}
}
baseBit0 := pilosa.ShardWidth * shardNums[0]
baseBit1 := pilosa.ShardWidth * shardNums[1]
baseBit2 := pilosa.ShardWidth * shardNums[2]
maxShard := uint64(0)
for _, x := range shardNums {
if x > maxShard {
maxShard = x
}
}
_, err := c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_, err := c.GetNode(0).API.CreateIndex(context.Background(), c.Idx(), pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
_, err = c.GetNode(0).API.CreateField(context.Background(), c.Idx(), "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
hldr0.MustSetBits("i", "f", 100, baseBit0+10)
hldr0.MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12)
hldr0.MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
hldr0.MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
hldr0.MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
hldr0.MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2)
// Connect to each node to compare results.
client := make([]*Client, 3)
client[0] = MustNewClient(c.GetNode(0).URL(), pilosa.GetHTTPClient(nil))
client[1] = MustNewClient(c.GetNode(1).URL(), pilosa.GetHTTPClient(nil))
client[2] = MustNewClient(c.GetNode(2).URL(), pilosa.GetHTTPClient(nil))
hldr1.MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
hldr1.MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
hldr1.MustSetBits("i", "f", 98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
hldr1.MustSetBits("i", "f", 1, baseBit1+4)
hldr1.MustSetBits("i", "f", 22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
b0 := uint64(ShardWidth * 0)
b1 := uint64(ShardWidth * 1)
b2 := uint64(ShardWidth * 2)
hldr2.MustSetBits("i", "f", 24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
hldr2.MustSetBits("i", "f", 20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
hldr2.MustSetBits("i", "f", 21, baseBit2+10)
hldr2.MustSetBits("i", "f", 100, baseBit2+10)
hldr2.MustSetBits("i", "f", 99, baseBit2+10, baseBit2+11, baseBit2+12)
hldr2.MustSetBits("i", "f", 98, baseBit2+10, baseBit2+11)
hldr2.MustSetBits("i", "f", 22, baseBit2+10, baseBit2+11, baseBit2+12)
// helper to let us avoid repeating the b0+, etc, over and over
collate := func(a, b, c []uint64) []uint64 {
d := make([]uint64, len(a)+len(b)+len(c))
n := 0
for _, v := range a {
d[n] = b0 + v
n++
}
for _, v := range b {
d[n] = b1 + v
n++
}
for _, v := range c {
d[n] = b2 + v
n++
}
return d
}
// data set. part of the goal of this is to have different top counts using a single node than
// using all three nodes
rows := map[uint64][]uint64{
100: collate([]uint64{10}, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, []uint64{10}),
4: collate([]uint64{10, 11, 12, 13, 14, 15}, nil, nil),
2: collate([]uint64{1, 2, 3, 4}, nil, nil),
3: collate([]uint64{1, 2, 3, 4, 5}, nil, nil),
99: collate(nil, []uint64{1, 2, 3, 4}, []uint64{10, 11, 12}),
98: collate(nil, []uint64{1, 2, 3, 4, 5, 6}, []uint64{10, 11}),
22: collate([]uint64{1, 2}, []uint64{1, 2, 3, 4, 5}, []uint64{10, 11, 12}),
1: collate(nil, []uint64{4}, nil),
21: collate(nil, nil, []uint64{10}),
}
bits := 0
for _, v := range rows {
bits += len(v)
}
rowIDs := make([]uint64, bits)
colIDs := make([]uint64, bits)
n := 0
for k, cols := range rows {
for _, v := range cols {
rowIDs[n] = k
colIDs[n] = v
n++
}
}
req := &pilosa.ImportRequest{
Index: c.Idx(),
Field: "f",
RowIDs: rowIDs,
ColumnIDs: colIDs,
Shard: ^uint64(0),
}
err = client[0].Import(context.Background(), nil, req, &pilosa.ImportOptions{})
if err != nil {
t.Fatalf("importing data: %v", err)
}
// Rebuild the RankCache.
// We have to do this to avoid the 10-second cache invalidation delay
// built into cache.Invalidate()
@ -124,19 +124,13 @@ func TestClient_MultiNode(t *testing.T) {
t.Fatalf("recalculating cache: %v", err)
}
// Connect to each node to compare results.
client := make([]*Client, 3)
client[0] = MustNewClient(c.GetNode(0).URL(), pilosa.GetHTTPClient(nil))
client[1] = MustNewClient(c.GetNode(1).URL(), pilosa.GetHTTPClient(nil))
client[2] = MustNewClient(c.GetNode(2).URL(), pilosa.GetHTTPClient(nil))
topN := 4
queryRequest := &pilosa.QueryRequest{
Query: fmt.Sprintf(`TopN(f, n=%d)`, topN),
Remote: false,
}
result, err := client[0].Query(context.Background(), "i", queryRequest)
result, err := client[0].Query(context.Background(), c.Idx(), queryRequest)
if err != nil {
t.Fatal(err)
}
@ -157,11 +151,11 @@ func TestClient_MultiNode(t *testing.T) {
t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
}
result1, err := client[1].Query(context.Background(), "i", queryRequest)
result1, err := client[1].Query(context.Background(), c.Idx(), queryRequest)
if err != nil {
t.Fatal(err)
}
result2, err := client[2].Query(context.Background(), "i", queryRequest)
result2, err := client[2].Query(context.Background(), c.Idx(), queryRequest)
if err != nil {
t.Fatal(err)
}
@ -181,16 +175,18 @@ func TestClient_Export(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
cmd := cluster.GetNode(0)
keyed := cluster.Idx("k")
unkeyed := cluster.Idx("u")
host := cmd.URL()
cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true})
cmd.MustCreateIndex(t, "unkeyed", pilosa.IndexOptions{Keys: false})
cmd.MustCreateIndex(t, keyed, pilosa.IndexOptions{Keys: true})
cmd.MustCreateIndex(t, unkeyed, pilosa.IndexOptions{Keys: false})
cmd.MustCreateField(t, "keyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
cmd.MustCreateField(t, "keyed", "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000))
cmd.MustCreateField(t, "unkeyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
cmd.MustCreateField(t, "unkeyed", "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000))
cmd.MustCreateField(t, keyed, "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
cmd.MustCreateField(t, keyed, "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000))
cmd.MustCreateField(t, unkeyed, "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
cmd.MustCreateField(t, unkeyed, "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
data := []pilosa.Bit{
@ -207,7 +203,7 @@ func TestClient_Export(t *testing.T) {
t.Run("Export unkeyed,unkeyedf", func(t *testing.T) {
// Populate data.
for _, bit := range data {
_, err := c.Query(context.Background(), "unkeyed", &pilosa.QueryRequest{
_, err := c.Query(context.Background(), unkeyed, &pilosa.QueryRequest{
Query: fmt.Sprintf(`Set(%d, unkeyedf=%d)`, bit.ColumnID, bit.RowID),
Remote: false,
})
@ -220,7 +216,7 @@ func TestClient_Export(t *testing.T) {
bw := bufio.NewWriter(buf)
// Send export request.
if err := c.ExportCSV(context.Background(), "unkeyed", "unkeyedf", 0, bw); err != nil {
if err := c.ExportCSV(context.Background(), unkeyed, "unkeyedf", 0, bw); err != nil {
t.Fatal(err)
}
@ -241,7 +237,7 @@ func TestClient_Export(t *testing.T) {
t.Run("Export unkeyed,keyedf", func(t *testing.T) {
// Populate data.
for _, bit := range data {
_, err := c.Query(context.Background(), "unkeyed", &pilosa.QueryRequest{
_, err := c.Query(context.Background(), unkeyed, &pilosa.QueryRequest{
Query: fmt.Sprintf(`Set(%d, keyedf=%s)`, bit.ColumnID, bit.RowKey),
Remote: false,
})
@ -254,7 +250,7 @@ func TestClient_Export(t *testing.T) {
bw := bufio.NewWriter(buf)
// Send export request.
if err := c.ExportCSV(context.Background(), "unkeyed", "keyedf", 0, bw); err != nil {
if err := c.ExportCSV(context.Background(), unkeyed, "keyedf", 0, bw); err != nil {
t.Fatal(err)
}
@ -275,7 +271,7 @@ func TestClient_Export(t *testing.T) {
t.Run("Export keyed,unkeyedf", func(t *testing.T) {
// Populate data.
for _, bit := range data {
_, err := c.Query(context.Background(), "keyed", &pilosa.QueryRequest{
_, err := c.Query(context.Background(), keyed, &pilosa.QueryRequest{
Query: fmt.Sprintf(`Set("%s", unkeyedf=%d)`, bit.ColumnKey, bit.RowID),
Remote: false,
})
@ -289,34 +285,35 @@ func TestClient_Export(t *testing.T) {
// Send export request for every partition.
for i := 0; i < disco.DefaultPartitionN; i++ {
if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", uint64(i), bw); err != nil {
if err := c.ExportCSV(context.Background(), keyed, "unkeyedf", uint64(i), bw); err != nil {
t.Fatal(err)
}
}
got := buf.String()
gotSlice := strings.Split(got, "\n")
// Expected output is not sorted because of key sharding.
exp := "" +
"2,col200\n" +
"2,col201\n" +
"2,col202\n" +
"2,col203\n" +
"1,col103\n" +
"1,col102\n" +
"1,col101\n" +
"1,col100\n"
// Verify data.
if got != exp {
t.Fatalf("unexpected export data: %q, expected %q", got, exp)
expSlice := []string{
"1,col103",
"1,col102",
"1,col101",
"1,col100",
"2,col200",
"2,col201",
"2,col202",
"2,col203",
"",
}
if !sameStringSlice(gotSlice, expSlice) {
t.Fatalf("unexpected results: %q", gotSlice)
}
})
t.Run("Export keyed,keyedf", func(t *testing.T) {
// Populate data.
for _, bit := range data {
_, err := c.Query(context.Background(), "keyed", &pilosa.QueryRequest{
_, err := c.Query(context.Background(), keyed, &pilosa.QueryRequest{
Query: fmt.Sprintf(`Set("%s", keyedf=%s)`, bit.ColumnKey, bit.RowKey),
Remote: false,
})
@ -330,27 +327,28 @@ func TestClient_Export(t *testing.T) {
// Send export request.
for i := 0; i < disco.DefaultPartitionN; i++ {
if err := c.ExportCSV(context.Background(), "keyed", "keyedf", uint64(i), bw); err != nil {
if err := c.ExportCSV(context.Background(), keyed, "keyedf", uint64(i), bw); err != nil {
t.Fatal(err)
}
}
got := buf.String()
gotSlice := strings.Split(got, "\n")
// Expected output is unsorted because of key sharding.
exp := "" +
"row2,col200\n" +
"row2,col201\n" +
"row2,col202\n" +
"row2,col203\n" +
"row1,col103\n" +
"row1,col102\n" +
"row1,col101\n" +
"row1,col100\n"
// Verify data.
if got != exp {
t.Fatalf("unexpected export data: %q, expected %q", got, exp)
// Expected output is not sorted because of key sharding.
expSlice := []string{
"row1,col103",
"row1,col102",
"row1,col101",
"row1,col100",
"row2,col200",
"row2,col201",
"row2,col202",
"row2,col203",
"",
}
if !sameStringSlice(gotSlice, expSlice) {
t.Fatalf("unexpected results: %q", gotSlice)
}
})
}
@ -363,16 +361,18 @@ func TestClient_Import(t *testing.T) {
cmd := cluster.GetNode(0)
host := cmd.URL()
api := cmd.API
keyed := cluster.Idx("k")
unkeyed := cluster.Idx("u")
cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true})
cmd.MustCreateIndex(t, "unkeyed", pilosa.IndexOptions{Keys: false})
cmd.MustCreateIndex(t, keyed, pilosa.IndexOptions{Keys: true})
cmd.MustCreateIndex(t, unkeyed, pilosa.IndexOptions{Keys: false})
cmd.MustCreateField(t, "keyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0), pilosa.OptFieldKeys())
cmd.MustCreateField(t, "keyed", "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0))
cmd.MustCreateField(t, "unkeyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0), pilosa.OptFieldKeys())
cmd.MustCreateField(t, "unkeyed", "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0))
cmd.MustCreateField(t, keyed, "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0), pilosa.OptFieldKeys())
cmd.MustCreateField(t, keyed, "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0))
cmd.MustCreateField(t, unkeyed, "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0), pilosa.OptFieldKeys())
cmd.MustCreateField(t, unkeyed, "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0))
indexes := map[bool]string{true: "keyed", false: "unkeyed"}
indexes := map[bool]string{true: keyed, false: unkeyed}
fields := map[bool]string{true: "keyedf", false: "unkeyedf"}
recKeys := []string{"rec-a", "rec-b", "rec-c"}
@ -427,13 +427,13 @@ func TestClient_Import(t *testing.T) {
}
}
for keyed, indexName := range indexes {
for useKeys, indexName := range indexes {
for _, fieldName := range fields {
req := pilosa.ImportRequest{
Index: indexName,
Field: fieldName,
}
if indexName == "keyed" {
if indexName == keyed {
req.ColumnKeys = recKeys
req.Shard = ^uint64(0)
} else {
@ -462,14 +462,14 @@ func TestClient_Import(t *testing.T) {
if err != nil {
t.Fatal(err)
}
checkResults(results, keyed, 3)
checkResults(results, useKeys, 3)
// Now clear a bit...
req = pilosa.ImportRequest{
Index: indexName,
Field: fieldName,
}
if indexName == "keyed" {
if indexName == keyed {
req.ColumnKeys = recKeys[2:]
req.Shard = ^uint64(0)
} else {
@ -504,32 +504,33 @@ func TestClient_Import(t *testing.T) {
if err != nil {
t.Fatal(err)
}
checkResults(results, keyed, 2)
checkResults(results, useKeys, 2)
}
}
}
// Ensure client can bulk import data.
func TestClient_ImportRoaring(t *testing.T) {
cluster := test.MustRunCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerReplicaN(3))},
[]server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerReplicaN(3))},
[]server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerReplicaN(3))},
)
cluster := test.MustUnsharedCluster(t, 3)
// Unshared because we want to set ReplicaN = 3 so we can verify data present on all nodes
for _, c := range cluster.Nodes {
c.Config.Cluster.ReplicaN = 3
}
err := cluster.Start()
if err != nil {
t.Fatalf("starting cluster: %v", err)
}
defer cluster.Close()
_, err := cluster.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_, err = cluster.GetNode(0).API.CreateIndex(context.Background(), cluster.Idx(), pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = cluster.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
_, err = cluster.GetNode(0).API.CreateField(context.Background(), cluster.Idx(), "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
_, err = cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"})
_, err = cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: cluster.Idx(), Query: "Set(0, f=1)"})
if err != nil {
t.Fatalf("querying: %v", err)
}
@ -539,101 +540,101 @@ func TestClient_ImportRoaring(t *testing.T) {
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100")
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, cluster.Idx(), "f", 0, false, roaringReq); err != nil {
t.Fatal(err)
}
hldr := test.Holder{Holder: cluster.GetNode(0).Server.Holder()}
// Verify data on node 0.
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
if a := hldr.Row(cluster.Idx(), "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
t.Fatalf("unexpected columns: %+v", a)
}
if a := hldr.Row("i", "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
if a := hldr.Row(cluster.Idx(), "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected columns: %+v", a)
}
hldr2 := test.Holder{Holder: cluster.GetNode(1).Server.Holder()}
// Verify data on node 1.
if a := hldr2.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
if a := hldr2.Row(cluster.Idx(), "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
t.Fatalf("unexpected columns: %+v", a)
}
if a := hldr2.Row("i", "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
if a := hldr2.Row(cluster.Idx(), "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected columns: %+v", a)
}
// Ensure that sending a roaring import with the clear flag works as expected.
// [65539, 65540]
roaringReq = makeImportRoaringRequest(true, "3A30000001000000010001001000000003000400")
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, cluster.Idx(), "f", 0, false, roaringReq); err != nil {
t.Fatal(err)
}
// Verify data on node 0.
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
if a := hldr.Row(cluster.Idx(), "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
if a := hldr.Row("i", "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
if a := hldr.Row(cluster.Idx(), "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
// Verify data on node 1.
if a := hldr2.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
if a := hldr2.Row(cluster.Idx(), "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
if a := hldr2.Row("i", "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
if a := hldr2.Row(cluster.Idx(), "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
// Ensure that sending a roaring import with the clear flag works as expected.
// [4, 6, 65537, 65539]
roaringReq = makeImportRoaringRequest(true, "3A300000020000000000010001000100180000001C0000000400060001000300")
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, cluster.Idx(), "f", 0, false, roaringReq); err != nil {
t.Fatal(err)
}
// Verify data on node 0.
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 5, 7, 8, 9, 10}) {
if a := hldr.Row(cluster.Idx(), "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 5, 7, 8, 9, 10}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
if a := hldr.Row("i", "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
if a := hldr.Row(cluster.Idx(), "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
// Verify data on node 1.
if a := hldr2.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 5, 7, 8, 9, 10}) {
if a := hldr2.Row(cluster.Idx(), "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 5, 7, 8, 9, 10}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
if a := hldr2.Row("i", "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
if a := hldr2.Row(cluster.Idx(), "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
// Ensure that sending a roaring import with the clear flag works as expected.
// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
roaringReq = makeImportRoaringRequest(true, "3B3001000100000900010000000100010009000100")
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, cluster.Idx(), "f", 0, false, roaringReq); err != nil {
t.Fatal(err)
}
// Verify data on node 0.
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{}) {
if a := hldr.Row(cluster.Idx(), "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
if a := hldr.Row("i", "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
if a := hldr.Row(cluster.Idx(), "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
// Verify data on node 1.
if a := hldr2.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{}) {
if a := hldr2.Row(cluster.Idx(), "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
if a := hldr2.Row("i", "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
if a := hldr2.Row(cluster.Idx(), "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected clear columns: %+v", a)
}
}
// Ensure client can bulk import data with multiple views and not deadlock.
func TestClient_ImportRoaring_MultiView(t *testing.T) {
cluster := test.MustNewCluster(t, 2)
cluster := test.MustUnsharedCluster(t, 2)
for _, c := range cluster.Nodes {
c.Config.Cluster.ReplicaN = 2
}
@ -645,15 +646,15 @@ func TestClient_ImportRoaring_MultiView(t *testing.T) {
api := cluster.GetNode(0).API
_, err = api.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_, err = api.CreateIndex(context.Background(), cluster.Idx(), pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = api.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
_, err = api.CreateField(context.Background(), cluster.Idx(), "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
_, err = api.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"})
_, err = api.Query(context.Background(), &pilosa.QueryRequest{Index: cluster.Idx(), Query: "Set(0, f=1)"})
if err != nil {
t.Fatalf("querying: %v", err)
}
@ -664,7 +665,7 @@ func TestClient_ImportRoaring_MultiView(t *testing.T) {
req := &pilosa.ImportRoaringRequest{Views: map[string][]byte{}}
req.Views["a"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100")
req.Views["b"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100")
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, req); err != nil {
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, cluster.Idx(), "f", 0, false, req); err != nil {
t.Fatal(err)
}
}
@ -676,18 +677,20 @@ func TestClient_ImportKeys(t *testing.T) {
defer cluster.Close()
cmd := cluster.GetNode(0)
host := cmd.URL()
keyed := cluster.Idx("k")
unkeyed := cluster.Idx("u")
cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true})
cmd.MustCreateIndex(t, "unkeyed", pilosa.IndexOptions{Keys: false})
cmd.MustCreateIndex(t, keyed, pilosa.IndexOptions{Keys: true})
cmd.MustCreateIndex(t, unkeyed, pilosa.IndexOptions{Keys: false})
cmd.MustCreateField(t, "keyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
cmd.MustCreateField(t, "keyed", "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000))
cmd.MustCreateField(t, "unkeyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
cmd.MustCreateField(t, keyed, "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
cmd.MustCreateField(t, keyed, "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000))
cmd.MustCreateField(t, unkeyed, "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
// Send import request.
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
baseReq := &pilosa.ImportRequest{
Index: "keyed",
Index: keyed,
Field: "keyedf",
ColumnKeys: []string{"eve", "alice", "bob", "eve", "alice", "eve"},
ColumnIDs: []uint64{1, 2, 3, 1, 2, 1},
@ -697,14 +700,14 @@ func TestClient_ImportKeys(t *testing.T) {
t.Run("Import keyed,keyed", func(t *testing.T) {
req := baseReq.Clone()
req.Index = "keyed"
req.Index = keyed
req.Field = "keyedf"
req.ColumnIDs, req.RowIDs = nil, nil
if err := c.Import(context.Background(), nil, req, &pilosa.ImportOptions{}); err != nil {
t.Fatal(err)
}
resp := cmd.QueryAPI(t, &pilosa.QueryRequest{
Index: "keyed",
Index: keyed,
Query: "TopN(keyedf)",
})
if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
@ -720,14 +723,14 @@ func TestClient_ImportKeys(t *testing.T) {
t.Run("Import keyed,unkeyedf", func(t *testing.T) {
req := baseReq.Clone()
req.Index = "keyed"
req.Index = keyed
req.Field = "unkeyedf"
req.ColumnIDs, req.RowKeys = nil, nil
if err := c.Import(context.Background(), nil, req, &pilosa.ImportOptions{}); err != nil {
t.Fatal(err)
}
resp := cmd.QueryAPI(t, &pilosa.QueryRequest{
Index: "keyed",
Index: keyed,
Query: "TopN(unkeyedf)",
})
if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
@ -743,14 +746,14 @@ func TestClient_ImportKeys(t *testing.T) {
t.Run("Import unkeyed,keyed", func(t *testing.T) {
req := baseReq.Clone()
req.Index = "unkeyed"
req.Index = unkeyed
req.Field = "keyedf"
req.ColumnKeys, req.RowIDs = nil, nil
if err := c.Import(context.Background(), nil, req, &pilosa.ImportOptions{}); err != nil {
t.Fatal(err)
}
resp := cmd.QueryAPI(t, &pilosa.QueryRequest{
Index: "unkeyed",
Index: unkeyed,
Query: "TopN(keyedf)",
})
if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
@ -772,10 +775,11 @@ func TestClient_ImportKeys(t *testing.T) {
cmd1 := cluster.GetNode(1)
host0 := cmd0.URL()
host1 := cmd1.URL()
keyed := cluster.Idx("k")
cmd0.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true})
cmd0.MustCreateField(t, "keyed", "keyedf0", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
cmd0.MustCreateField(t, "keyed", "keyedf1", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
cmd0.MustCreateIndex(t, keyed, pilosa.IndexOptions{Keys: true})
cmd0.MustCreateField(t, keyed, "keyedf0", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
cmd0.MustCreateField(t, keyed, "keyedf1", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
// Send import request.
c0 := MustNewClient(host0, pilosa.GetHTTPClient(nil))
@ -784,7 +788,7 @@ func TestClient_ImportKeys(t *testing.T) {
// Import to node0.
t.Run("Import node0", func(t *testing.T) {
req := &pilosa.ImportRequest{
Index: "keyed",
Index: keyed,
Field: "keyedf0",
ColumnKeys: []string{"eve", "alice", "bob", "eve", "alice", "eve"},
RowKeys: []string{"green", "green", "green", "blue", "blue", "purple"},
@ -793,7 +797,7 @@ func TestClient_ImportKeys(t *testing.T) {
t.Fatal(err)
}
resp := cmd0.QueryAPI(t, &pilosa.QueryRequest{
Index: "keyed",
Index: keyed,
Query: "TopN(keyedf0)",
})
if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
@ -810,7 +814,7 @@ func TestClient_ImportKeys(t *testing.T) {
// Import to node1 (ensure import is routed to primary for translation).
t.Run("Import node1", func(t *testing.T) {
req := &pilosa.ImportRequest{
Index: "keyed",
Index: keyed,
Field: "keyedf1",
ColumnKeys: []string{"eve", "alice", "bob", "eve", "alice", "eve"},
RowKeys: []string{"green", "green", "green", "blue", "blue", "purple"},
@ -823,7 +827,7 @@ func TestClient_ImportKeys(t *testing.T) {
time.Sleep(500 * time.Millisecond)
resp := cmd1.QueryAPI(t, &pilosa.QueryRequest{
Index: "keyed",
Index: keyed,
Query: "TopN(keyedf1)",
})
if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
@ -849,7 +853,7 @@ func TestClient_ImportKeys(t *testing.T) {
fldName := "f"
// Load bitmap into cache to ensure cache gets updated.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
index := hldr.MustCreateIndexIfNotExists(cluster.Idx(), pilosa.IndexOptions{Keys: true})
_, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
if err != nil {
t.Fatal(err)
@ -858,7 +862,7 @@ func TestClient_ImportKeys(t *testing.T) {
// Send import request.
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
req := &pilosa.ImportValueRequest{
Index: "i",
Index: cluster.Idx(),
Field: "f",
ColumnKeys: []string{"col1", "col2", "col3"},
Values: []int64{-10, 20, 40},
@ -873,7 +877,7 @@ func TestClient_ImportKeys(t *testing.T) {
Remote: false,
}
result, err := c.Query(context.Background(), "i", queryRequest)
result, err := c.Query(context.Background(), cluster.Idx(), queryRequest)
if err != nil {
t.Fatal(err)
}
@ -884,7 +888,7 @@ func TestClient_ImportKeys(t *testing.T) {
// Clear data.
req = &pilosa.ImportValueRequest{
Index: "i",
Index: cluster.Idx(),
Field: "f",
ColumnKeys: []string{"col2"},
Values: []int64{20},
@ -899,7 +903,7 @@ func TestClient_ImportKeys(t *testing.T) {
Remote: false,
}
result, err = c.Query(context.Background(), "i", queryRequest)
result, err = c.Query(context.Background(), cluster.Idx(), queryRequest)
if err != nil {
t.Fatal(err)
}
@ -924,7 +928,7 @@ func TestClient_ImportIDs(t *testing.T) {
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
idxName := "i"
idxName := cluster.Idx()
fldName := "f"
// Load bitmap into cache to ensure cache gets updated.
@ -996,7 +1000,7 @@ func TestClient_ImportValue(t *testing.T) {
fldName := "f"
// Load bitmap into cache to ensure cache gets updated.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
index := hldr.MustCreateIndexIfNotExists(cluster.Idx(), pilosa.IndexOptions{})
_, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
if err != nil {
t.Fatal(err)
@ -1005,7 +1009,7 @@ func TestClient_ImportValue(t *testing.T) {
// Send import request.
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
req := &pilosa.ImportValueRequest{
Index: "i",
Index: cluster.Idx(),
Field: "f",
ColumnIDs: []uint64{1, 2, 3},
Values: []int64{-10, 20, 40},
@ -1015,7 +1019,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Sum.
if resp, err := c.Query(context.Background(), "i", &pilosa.QueryRequest{Query: `Sum(field=f)`}); err != nil {
if resp, err := c.Query(context.Background(), cluster.Idx(), &pilosa.QueryRequest{Query: `Sum(field=f)`}); err != nil {
t.Fatal(err)
} else if vc, ok := resp.Results[0].(pilosa.ValCount); !ok {
t.Fatalf("expected ValCount; got %T", resp.Results[0])
@ -1024,7 +1028,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Max.
if resp, err := c.Query(context.Background(), "i", &pilosa.QueryRequest{Query: `Max(field=f)`}); err != nil {
if resp, err := c.Query(context.Background(), cluster.Idx(), &pilosa.QueryRequest{Query: `Max(field=f)`}); err != nil {
t.Fatal(err)
} else if vc, ok := resp.Results[0].(pilosa.ValCount); !ok {
t.Fatalf("expected ValCount; got %T", resp.Results[0])
@ -1037,14 +1041,14 @@ func TestClient_ImportValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
preIUsage, err := c.GetIndexUsage(context.Background(), "i")
preIUsage, err := c.GetIndexUsage(context.Background(), cluster.Idx())
if err != nil {
t.Fatal(err)
}
// Send import request.
req = &pilosa.ImportValueRequest{
Index: "i",
Index: cluster.Idx(),
Field: "f",
ColumnIDs: []uint64{1, 3},
Values: []int64{-10, 40},
@ -1058,7 +1062,7 @@ func TestClient_ImportValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
postIUsage, err := c.GetIndexUsage(context.Background(), "i")
postIUsage, err := c.GetIndexUsage(context.Background(), cluster.Idx())
if err != nil {
t.Fatal(err)
}
@ -1076,7 +1080,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Sum.
if resp, err := c.Query(context.Background(), "i", &pilosa.QueryRequest{Query: `Sum(field=f)`}); err != nil {
if resp, err := c.Query(context.Background(), cluster.Idx(), &pilosa.QueryRequest{Query: `Sum(field=f)`}); err != nil {
t.Fatal(err)
} else if vc, ok := resp.Results[0].(pilosa.ValCount); !ok {
t.Fatalf("expected ValCount; got %T", resp.Results[0])
@ -1085,7 +1089,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Max.
if resp, err := c.Query(context.Background(), "i", &pilosa.QueryRequest{Query: `Max(field=f)`}); err != nil {
if resp, err := c.Query(context.Background(), cluster.Idx(), &pilosa.QueryRequest{Query: `Max(field=f)`}); err != nil {
t.Fatal(err)
} else if vc, ok := resp.Results[0].(pilosa.ValCount); !ok {
t.Fatalf("expected ValCount; got %T", resp.Results[0])
@ -1104,7 +1108,7 @@ func TestClient_ImportExistence(t *testing.T) {
hldr := test.Holder{Holder: holder}
t.Run("Set", func(t *testing.T) {
idxName := "iset"
idxName := cluster.Idx("s")
fldName := "fset"
index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackExistence: true})
@ -1116,7 +1120,7 @@ func TestClient_ImportExistence(t *testing.T) {
// Send import request.
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
req := &pilosa.ImportRequest{
Index: "iset",
Index: cluster.Idx("s"),
Field: "fset",
ColumnIDs: []uint64{1, 5, 6},
RowIDs: []uint64{0, 0, 200},
@ -1140,7 +1144,7 @@ func TestClient_ImportExistence(t *testing.T) {
})
t.Run("Int", func(t *testing.T) {
idxName := "iint"
idxName := cluster.Idx("i")
fldName := "fint"
index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackExistence: true})
@ -1152,7 +1156,7 @@ func TestClient_ImportExistence(t *testing.T) {
// Send import request.
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
req := &pilosa.ImportValueRequest{
Index: "iint",
Index: cluster.Idx("i"),
Field: "fint",
ColumnIDs: []uint64{1, 2, 3},
Values: []int64{-10, 20, 40},
@ -1186,13 +1190,13 @@ func TestClient_FragmentBlocks(t *testing.T) {
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100)
hldr.SetBit(cluster.Idx(), "f", 0, 1)
hldr.SetBit(cluster.Idx(), "f", pilosa.HashBlockSize*3, 100)
// Set a bit on a different shard.
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit(cluster.Idx(), "f", 0, 1)
c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil))
blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", "standard", 0)
blocks, err := c.FragmentBlocks(context.Background(), nil, cluster.Idx(), "f", "standard", 0)
if err != nil {
t.Fatal(err)
} else if len(blocks) != 2 {
@ -1204,7 +1208,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
}
// Verify data matches local blocks.
if a, err := cmd.API.FragmentBlocks(context.Background(), "i", "f", "standard", 0); err != nil {
if a, err := cmd.API.FragmentBlocks(context.Background(), cluster.Idx(), "f", "standard", 0); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(a, blocks) {
t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks))
@ -1238,7 +1242,7 @@ func TestClient_CreateTimeField(t *testing.T) {
c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil))
index := "cdf"
index := cluster.Idx()
err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
@ -1278,7 +1282,7 @@ func TestClient_CreateDecimalField(t *testing.T) {
c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil))
index := "cdf"
index := cluster.Idx()
err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
@ -1575,19 +1579,15 @@ func TestClient_ServerInfoHasBackend(t *testing.T) {
pilosa.MustBackendToTxtype(si.StorageBackend) // panics if invalid
}
func TestClient_ImportRoaringExists(t *testing.T) {
cluster := test.MustNewCluster(t, 1)
err := cluster.Start()
if err != nil {
t.Fatalf("starting cluster: %v", err)
}
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
node := cluster.GetNode(0)
_, err = node.API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{TrackExistence: true})
_, err := node.API.CreateIndex(context.Background(), cluster.Idx(), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = node.API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
_, err = node.API.CreateField(context.Background(), cluster.Idx(), "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -1597,10 +1597,10 @@ func TestClient_ImportRoaringExists(t *testing.T) {
// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100")
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, cluster.Idx(), "f", 0, false, roaringReq); err != nil {
t.Fatal(err)
}
qr, err := node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"})
qr, err := node.API.Query(context.Background(), &pilosa.QueryRequest{Index: cluster.Idx(), Query: "All()"})
if err != nil {
t.Fatalf(" %v ", err)
}
@ -1609,12 +1609,12 @@ func TestClient_ImportRoaringExists(t *testing.T) {
t.Fatalf(" Row unexpected columns: got %+v expected: %+v", got, []uint64{})
}
roaringReq.UpdateExistence = true
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, cluster.Idx(), "f", 0, false, roaringReq); err != nil {
t.Fatal(err)
}
expected := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}
qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"})
qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: cluster.Idx(), Query: "All()"})
if err != nil {
t.Fatalf("Query error: %+v", err)
}

32
server/handler_test.go
View file

@ -27,7 +27,7 @@ import (
)
func TestHandler_PostSchemaCluster(t *testing.T) {
cluster := test.MustRunCluster(t, 3)
cluster := test.MustRunUnsharedCluster(t, 3)
defer cluster.Close()
cmd := cluster.GetNode(0)
h := cmd.Handler.(*pilosa.Handler).Handler
@ -66,7 +66,8 @@ func TestHandler_PostSchemaCluster(t *testing.T) {
}
func TestHandler_Endpoints(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
// this test is full of hardcoded indexes and things
cluster := test.MustRunUnsharedCluster(t, 1)
defer cluster.Close()
cmd := cluster.GetNode(0)
h := cmd.Handler.(*pilosa.Handler).Handler
@ -175,11 +176,8 @@ func TestHandler_Endpoints(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
const shard = 0
tx0, err := holder.BeginTx(true, i0.Index, shard)
if err != nil {
t.Fatal(err)
}
defer tx0.Rollback()
tx0 := holder.Txf().NewWritableQcx()
defer tx0.Abort()
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx0, 0, 0, nil); err != nil {
@ -188,22 +186,19 @@ func TestHandler_Endpoints(t *testing.T) {
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
if err := tx0.Commit(); err != nil {
if err := tx0.Finish(); err != nil {
t.Fatal(err)
}
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
tx1, err := holder.BeginTx(true, i1.Index, shard)
if err != nil {
t.Fatal(err)
}
defer tx1.Rollback()
tx1 := holder.Txf().NewWritableQcx()
defer tx1.Abort()
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx1, 0, 0, nil); err != nil {
t.Fatal(err)
}
if err := tx1.Commit(); err != nil {
if err := tx1.Finish(); err != nil {
t.Fatal(err)
}
@ -241,11 +236,8 @@ func TestHandler_Endpoints(t *testing.T) {
// i2 is for SchemaDetails
i2 := hldr.MustCreateIndexIfNotExists("i2", pilosa.IndexOptions{})
tx2, err := holder.BeginTx(true, i2.Index, shard)
if err != nil {
t.Fatal(err)
}
defer tx2.Rollback()
tx2 := holder.Txf().NewWritableQcx()
defer tx2.Abort()
if f, err := i2.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000)); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
@ -290,7 +282,7 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatal(err)
}
if err := tx2.Commit(); err != nil {
if err := tx2.Finish(); err != nil {
t.Fatal(err)
}

34
server/server_test.go
View file

@ -236,14 +236,14 @@ func TestConcurrentFieldCreation(t *testing.T) {
}
api0 := node0.API
if _, err := api0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil {
if _, err := api0.CreateIndex(context.Background(), cluster.Idx(), pilosa.IndexOptions{}); err != nil {
t.Fatalf("creating index: %v", err)
}
eg := errgroup.Group{}
for i := 0; i < 100; i++ {
i := i
eg.Go(func() error {
if _, err := api0.CreateField(context.Background(), "i", fmt.Sprintf("f%d", i)); err != nil {
if _, err := api0.CreateField(context.Background(), cluster.Idx(), fmt.Sprintf("f%d", i)); err != nil {
return err
}
return nil
@ -388,10 +388,10 @@ func TestMain_RecalculateCaches(t *testing.T) {
// Create the schema.
client0 := cluster.GetNode(0).Client()
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
if err := client0.CreateIndex(context.Background(), cluster.Idx(), pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create index:", err)
}
if err := client0.CreateField(context.Background(), "i", "f"); err != nil {
if err := client0.CreateField(context.Background(), cluster.Idx(), "f"); err != nil {
t.Fatal("create field:", err)
}
@ -402,7 +402,7 @@ func TestMain_RecalculateCaches(t *testing.T) {
data = append(data, fmt.Sprintf(`Set(%d, f=%d)`, columnID, rowID))
}
}
if _, err := cluster.GetNode(0).Query(t, "i", "", strings.Join(data, "")); err != nil {
if _, err := cluster.GetNode(0).Query(t, cluster.Idx(), "", strings.Join(data, "")); err != nil {
t.Fatal("setting columns:", err)
}
@ -416,7 +416,7 @@ func TestMain_RecalculateCaches(t *testing.T) {
// Run a TopN query on all nodes. The result should be the same as the target.
for _, m := range cluster.Nodes {
res, err := m.Query(t, "i", "", `TopN(f)`)
res, err := m.Query(t, cluster.Idx(), "", `TopN(f)`)
if err != nil {
t.Fatal(err)
}
@ -499,7 +499,7 @@ func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
func TestClusteringNodesReplica1(t *testing.T) {
cluster := test.MustRunCluster(t, 3)
cluster := test.MustRunUnsharedCluster(t, 3)
defer cluster.Close()
if err := cluster.AwaitState(disco.ClusterStateNormal, 100*time.Millisecond); err != nil {
@ -565,7 +565,7 @@ func TestClusteringNodesReplica2(t *testing.T) {
// Because this test shuts down 2 nodes, it needs to start as a 5-node
// cluster in order to retain enough available nodes for raft leader
// election.
cluster := test.MustNewCluster(t, 5)
cluster := test.MustUnsharedCluster(t, 5)
for _, c := range cluster.Nodes {
c.Config.Cluster.ReplicaN = 2
}
@ -920,10 +920,11 @@ func TestClusterMinMaxSumDecimal(t *testing.T) {
defer cluster.Close()
cmd := cluster.GetNode(0)
cmd.MustCreateIndex(t, "testdec", pilosa.IndexOptions{Keys: true, TrackExistence: true})
cmd.MustCreateField(t, "testdec", "adec", pilosa.OptFieldTypeDecimal(2))
index := cluster.Idx("i")
cmd.MustCreateIndex(t, index, pilosa.IndexOptions{Keys: true, TrackExistence: true})
cmd.MustCreateField(t, index, "adec", pilosa.OptFieldTypeDecimal(2))
test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", `
test.Do(t, "POST", fmt.Sprintf("%s/index/%i/query", cmd.URL(), cluster), `
Set("a", adec=42.2)
Set("b", adec=11.12)
Set("c", adec=13.41)
@ -934,21 +935,21 @@ Set("g", adec=15.52)
Set("h", adec=100.22)
`)
result := test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Sum(field=adec)")
result := test.Do(t, "POST", fmt.Sprintf("%s/index/%i/query", cmd.URL(), cluster), "Sum(field=adec)")
if !strings.Contains(result.Body, `"decimalValue":305.59`) {
t.Fatalf("expected decimal sum of 305.59, but got: '%s'", result.Body)
} else if !strings.Contains(result.Body, `"count":8`) {
t.Fatalf("expected count 8, but got: '%s'", result.Body)
}
result = test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Max(field=adec)")
result = test.Do(t, "POST", fmt.Sprintf("%s/index/%i/query", cmd.URL(), cluster), "Max(field=adec)")
if !strings.Contains(result.Body, `"decimalValue":100.22`) {
t.Fatalf("expected decimal max of 100.22, but got: '%s'", result.Body)
} else if !strings.Contains(result.Body, `"count":1`) {
t.Fatalf("expected count 1, but got: '%s'", result.Body)
}
result = test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Min(field=adec)")
result = test.Do(t, "POST", fmt.Sprintf("%s/index/%i/query", cmd.URL(), cluster), "Min(field=adec)")
if !strings.Contains(result.Body, `"decimalValue":11.12`) {
t.Fatalf("expected decimal min of 11.12, but got: '%s'", result.Body)
} else if !strings.Contains(result.Body, `"count":1`) {
@ -984,7 +985,8 @@ func TestClusterCreatedAtRace(t *testing.T) {
}
for k := 0; k < iterations; k++ {
t.Run(fmt.Sprintf("run-%d", k), func(t *testing.T) {
cluster := test.MustRunCluster(t, 4)
// needs fresh cluster so the index creations are new
cluster := test.MustRunUnsharedCluster(t, 4)
defer cluster.Close()
for _, com := range cluster.Nodes {
@ -1041,7 +1043,7 @@ func TestClusterCreatedAtRace(t *testing.T) {
}
func TestClusterQueryCountInDegraded(t *testing.T) {
cluster := test.MustNewCluster(t, 3)
cluster := test.MustUnsharedCluster(t, 3)
for _, c := range cluster.Nodes {
c.Config.Cluster.ReplicaN = 2
}

4
server_test.go
View file

@ -22,7 +22,7 @@ func TestViewsRemovalTTL(t *testing.T) {
// Create a client
client := node.Client()
indexName := "i"
indexName := cluster.Idx()
// Create indexes and field with ttl lasting 24 hours
if err := client.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{TrackExistence: true}); err != nil && err != pilosa.ErrIndexExists {
@ -196,7 +196,7 @@ func TestViewsRemovalStandard(t *testing.T) {
// Create a client
client := node.Client()
indexName := "i"
indexName := cluster.Idx()
// Create indexes and field with ttl lasting 24 hours
if err := client.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{TrackExistence: true}); err != nil && err != pilosa.ErrIndexExists {

9
sql/handler_test.go
View file

@ -7,9 +7,9 @@ import (
"math"
"testing"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/sql"
"github.com/featurebasedb/featurebase/v3/test"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/sql"
"github.com/molecula/featurebase/v3/test"
"vitess.io/vitess/go/vt/sqlparser"
)
@ -34,6 +34,9 @@ func TestHandler(t *testing.T) {
}
func TestSelectHandler_MapSelect(t *testing.T) {
// it's load-bearing that this is the only cluster in this directory right
// now, because that allows us to hardcode index names and not immediately
// die.
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
api := cluster.GetNode(0).API

134
sql3/planner/executionplanner_test.go
View file

@ -22,7 +22,7 @@ func TestPlanner_Show(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
index, err := c.GetHolder(0).CreateIndex("i", pilosa.IndexOptions{TrackExistence: true})
index, err := c.GetHolder(0).CreateIndex(c.Idx("i"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -33,7 +33,7 @@ func TestPlanner_Show(t *testing.T) {
t.Fatal(err)
}
index2, err := c.GetHolder(0).CreateIndex("i2", pilosa.IndexOptions{TrackExistence: false})
index2, err := c.GetHolder(0).CreateIndex(c.Idx("l"), pilosa.IndexOptions{TrackExistence: false})
if err != nil {
t.Fatal(err)
}
@ -65,7 +65,7 @@ func TestPlanner_Show(t *testing.T) {
})
t.Run("ShowColumns", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SHOW COLUMNS FROM i`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SHOW COLUMNS FROM %i`, c))
if err != nil {
t.Fatal(err)
}
@ -94,7 +94,7 @@ func TestPlanner_Show(t *testing.T) {
})
t.Run("ShowColumns2", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SHOW COLUMNS FROM i2`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SHOW COLUMNS FROM %l`, c))
if err != nil {
t.Fatal(err)
}
@ -371,7 +371,7 @@ func TestPlanner_CoverCreateTable(t *testing.T) {
// Ensure the schema field options match the expected options.
for _, fld := range fields {
t.Run(fmt.Sprintf("Field:%s", fld.name), func(t *testing.T) {
// Field `_id` isn't returned from FeatureBase in the schema,
// Field fmt.Sprintf(`_id`, c) isn't returned from FeatureBase in the schema,
// but we do want to validate that its type is used to determine
// whether or not the table is keyed.
if fld.name == "_id" {
@ -532,7 +532,7 @@ func TestPlanner_AlterTable(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
index, err := c.GetHolder(0).CreateIndex("i", pilosa.IndexOptions{TrackExistence: true})
index, err := c.GetHolder(0).CreateIndex(c.Idx("i"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -546,7 +546,7 @@ func TestPlanner_AlterTable(t *testing.T) {
server := c.GetNode(0).Server
t.Run("AlterTableDrop", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, server, `alter table i drop column f`)
results, columns, err := sql_test.MustQueryRows(t, server, fmt.Sprintf(`alter table %i drop column f`, c))
if err != nil {
t.Fatal(err)
}
@ -560,7 +560,7 @@ func TestPlanner_AlterTable(t *testing.T) {
})
t.Run("AlterTableAdd", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, server, `alter table i add column f int`)
results, columns, err := sql_test.MustQueryRows(t, server, fmt.Sprintf(`alter table %i add column f int`, c))
if err != nil {
t.Fatal(err)
}
@ -575,7 +575,7 @@ func TestPlanner_AlterTable(t *testing.T) {
t.Run("AlterTableRename", func(t *testing.T) {
t.Skip("not yet implemented")
results, columns, err := sql_test.MustQueryRows(t, server, `alter table i rename column f to g`)
results, columns, err := sql_test.MustQueryRows(t, server, fmt.Sprintf(`alter table %i rename column f to g`, c))
if err != nil {
t.Fatal(err)
}
@ -593,7 +593,7 @@ func TestPlanner_DropTable(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
index, err := c.GetHolder(0).CreateIndex("i", pilosa.IndexOptions{TrackExistence: true})
index, err := c.GetHolder(0).CreateIndex(c.Idx("i"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -605,7 +605,7 @@ func TestPlanner_DropTable(t *testing.T) {
}
t.Run("DropTable", func(t *testing.T) {
_, _, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `DROP TABLE i`)
_, _, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`DROP TABLE %i`, c))
if err != nil {
t.Fatal(err)
}
@ -616,7 +616,7 @@ func TestPlanner_ExpressionsInSelectListParen(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
i0, err := c.GetHolder(0).CreateIndex(c.Idx("j"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -627,7 +627,7 @@ func TestPlanner_ExpressionsInSelectListParen(t *testing.T) {
t.Fatal(err)
}
i1, err := c.GetHolder(0).CreateIndex("i1", pilosa.IndexOptions{TrackExistence: true})
i1, err := c.GetHolder(0).CreateIndex(c.Idx("k"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -640,7 +640,7 @@ func TestPlanner_ExpressionsInSelectListParen(t *testing.T) {
// Populate with data.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i0",
Index: c.Idx("j"),
Query: `
Set(1, a=10)
Set(1, b=100)
@ -651,7 +651,7 @@ func TestPlanner_ExpressionsInSelectListParen(t *testing.T) {
}
t.Run("ParenOne", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT (a != b) = false, _id FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT (a != b) = false, _id FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -672,7 +672,7 @@ func TestPlanner_ExpressionsInSelectListParen(t *testing.T) {
})
t.Run("ParenTwo", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT (a != b) = (false), _id FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT (a != b) = (false), _id FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -697,7 +697,7 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
i0, err := c.GetHolder(0).CreateIndex(c.Idx("j"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -716,7 +716,7 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) {
// Populate with data.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i0",
Index: c.Idx("j"),
Query: `
Set(1, a=10)
Set(1, b=100)
@ -730,7 +730,7 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) {
}
t.Run("LiteralsBool", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT false = true, _id FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT false = true, _id FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -751,7 +751,7 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) {
})
t.Run("LiteralsInt", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT 1 + 2, _id FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT 1 + 2, _id FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -772,7 +772,7 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) {
})
t.Run("LiteralsID", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT _id + 2, _id FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT _id + 2, _id FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -793,7 +793,7 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) {
})
t.Run("LiteralsDecimal", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT d + 2.0, _id FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT d + 2.0, _id FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -814,7 +814,7 @@ func TestPlanner_ExpressionsInSelectListLiterals(t *testing.T) {
})
t.Run("LiteralsString", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT str || ' bar', _id FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT str || ' bar', _id FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -839,7 +839,7 @@ func TestPlanner_ExpressionsInSelectListCase(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
i0, err := c.GetHolder(0).CreateIndex(c.Idx("j"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -858,7 +858,7 @@ func TestPlanner_ExpressionsInSelectListCase(t *testing.T) {
// Populate with data.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i0",
Index: c.Idx("j"),
Query: `
Set(1, a=10)
Set(1, b=100)
@ -872,7 +872,7 @@ func TestPlanner_ExpressionsInSelectListCase(t *testing.T) {
}
t.Run("CaseWithBase", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT b, case b when 100 then 10 when 201 then 20 else 5 end, _id FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT b, case b when 100 then 10 when 201 then 20 else 5 end, _id FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -894,7 +894,7 @@ func TestPlanner_ExpressionsInSelectListCase(t *testing.T) {
})
t.Run("CaseWithNoBase", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT b, case when b = 100 then 10 when b = 201 then 20 else 5 end, _id FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT b, case when b = 100 then 10 when b = 201 then 20 else 5 end, _id FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -920,7 +920,7 @@ func TestPlanner_Select(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
i0, err := c.GetHolder(0).CreateIndex(c.Idx("j"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -931,7 +931,7 @@ func TestPlanner_Select(t *testing.T) {
t.Fatal(err)
}
i1, err := c.GetHolder(0).CreateIndex("i1", pilosa.IndexOptions{TrackExistence: true})
i1, err := c.GetHolder(0).CreateIndex(c.Idx("k"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -944,7 +944,7 @@ func TestPlanner_Select(t *testing.T) {
// Populate with data.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i0",
Index: c.Idx("j"),
Query: `
Set(1, a=10)
Set(1, b=100)
@ -955,7 +955,7 @@ func TestPlanner_Select(t *testing.T) {
}
t.Run("UnqualifiedColumns", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT a, b, _id FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT a, b, _id FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -977,7 +977,7 @@ func TestPlanner_Select(t *testing.T) {
})
t.Run("QualifiedTableRef", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT bar.a, bar.b, bar._id FROM i0 as bar`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT bar.a, bar.b, bar._id FROM %j as bar`, c))
if err != nil {
t.Fatal(err)
}
@ -999,7 +999,7 @@ func TestPlanner_Select(t *testing.T) {
})
t.Run("AliasedUnqualifiedColumns", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT a as foo, b as bar, _id as baz FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT a as foo, b as bar, _id as baz FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -1021,7 +1021,7 @@ func TestPlanner_Select(t *testing.T) {
})
t.Run("QualifiedColumns", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT i0._id, i0.a, i0.b FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT %j._id, %j.a, %j.b FROM %j`, c, c, c, c))
if err != nil {
t.Fatal(err)
}
@ -1043,7 +1043,7 @@ func TestPlanner_Select(t *testing.T) {
})
t.Run("UnqualifiedStar", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT * FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT * FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -1065,7 +1065,7 @@ func TestPlanner_Select(t *testing.T) {
})
t.Run("QualifiedStar", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT i0.* FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT %j.* FROM %j`, c, c))
if err != nil {
t.Fatal(err)
}
@ -1087,7 +1087,7 @@ func TestPlanner_Select(t *testing.T) {
})
t.Run("NoIdentifier", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT a, b FROM i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT a, b FROM %j`, c))
if err != nil {
t.Fatal(err)
}
@ -1108,7 +1108,7 @@ func TestPlanner_Select(t *testing.T) {
})
t.Run("ErrFieldNotFound", func(t *testing.T) {
_, err := c.GetNode(0).Server.CompileExecutionPlan(context.Background(), `SELECT xyz FROM i0`)
_, err := c.GetNode(0).Server.CompileExecutionPlan(context.Background(), fmt.Sprintf(`SELECT xyz FROM %j`, c))
if err == nil || !strings.Contains(err.Error(), `column 'xyz' not found`) {
t.Fatalf("unexpected error: %v", err)
}
@ -1119,7 +1119,7 @@ func TestPlanner_SelectOrderBy(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
i0, err := c.GetHolder(0).CreateIndex(c.Idx("j"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -1132,7 +1132,7 @@ func TestPlanner_SelectOrderBy(t *testing.T) {
// Populate with data.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i0",
Index: c.Idx("j"),
Query: `
Set(1, a=10)
Set(1, b=100)
@ -1143,7 +1143,7 @@ func TestPlanner_SelectOrderBy(t *testing.T) {
}
t.Run("OrderBy", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT a, b, _id FROM i0 order by a desc`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT a, b, _id FROM %j order by a desc`, c))
if err != nil {
t.Fatal(err)
}
@ -1169,7 +1169,7 @@ func TestPlanner_SelectSelectSource(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
i0, err := c.GetHolder(0).CreateIndex(c.Idx("j"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -1182,7 +1182,7 @@ func TestPlanner_SelectSelectSource(t *testing.T) {
// Populate with data.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i0",
Index: c.Idx("j"),
Query: `
Set(1, a=10)
Set(1, b=100)
@ -1193,7 +1193,7 @@ func TestPlanner_SelectSelectSource(t *testing.T) {
}
t.Run("ParenSource", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT a, b, _id FROM (select * from i0)`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT a, b, _id FROM (select * from %j)`, c))
if err != nil {
t.Fatal(err)
}
@ -1215,7 +1215,7 @@ func TestPlanner_SelectSelectSource(t *testing.T) {
})
t.Run("ParenSourceWithAlias", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT foo.a, b, _id FROM (select * from i0) as foo`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT foo.a, b, _id FROM (select * from %j) as foo`, c))
if err != nil {
t.Fatal(err)
}
@ -1241,7 +1241,7 @@ func TestPlanner_In(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
i0, err := c.GetHolder(0).CreateIndex(c.Idx("j"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -1250,7 +1250,7 @@ func TestPlanner_In(t *testing.T) {
t.Fatal(err)
}
i1, err := c.GetHolder(0).CreateIndex("i1", pilosa.IndexOptions{TrackExistence: true})
i1, err := c.GetHolder(0).CreateIndex(c.Idx("k"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -1263,7 +1263,7 @@ func TestPlanner_In(t *testing.T) {
// Populate with data.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i0",
Index: c.Idx("j"),
Query: `
Set(1, a=10)
Set(2, a=20)
@ -1273,7 +1273,7 @@ func TestPlanner_In(t *testing.T) {
}
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i1",
Index: c.Idx("k"),
Query: `
Set(1, parentid=1)
Set(1, x=100)
@ -1289,9 +1289,9 @@ func TestPlanner_In(t *testing.T) {
t.Run("Count", func(t *testing.T) {
t.Skip("Need to add join conditions to get this to pass")
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT i0._id, i0.a, i1._id, i1.parentid, i1.x FROM i0 INNER JOIN i1 ON i0._id = i1.parentid`)
//results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid`)
//results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT a FROM i0 where a = 20`) // SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT %j._id, %j.a, %k._id, %k.parentid, %k.x FROM %j INNER JOIN %k ON %j._id = %k.parentid`, c, c, c, c, c, c, c, c, c))
//results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT COUNT(*) FROM %j INNER JOIN %k ON %j._id = %k.parentid`, c, c, c, c))
//results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT a FROM %j where a = 20`, c)) // SELECT COUNT(*) FROM %j INNER JOIN %k ON %j._id = %k.parentid
if err != nil {
t.Fatal(err)
}
@ -1310,8 +1310,8 @@ func TestPlanner_In(t *testing.T) {
})
/*t.Run("Count", func(t *testing.T) {
//results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 where i0._id in (select distinct parentid from i1)`) // SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid
//results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT COUNT(*) FROM %j INNER JOIN %k ON %j._id = %k.parentid`, c, c, c, c))
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT COUNT(*) FROM %j where %j._id in (select distinct parentid from %k)`, c, c, c)) // SELECT COUNT(*) FROM %j INNER JOIN %k ON %j._id = %k.parentid
if err != nil {
t.Fatal(err)
}
@ -1330,7 +1330,7 @@ func TestPlanner_In(t *testing.T) {
})
t.Run("CountWithParentCondition", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 where i0._id in (select distinct parentid from i1) and i0.a = 10`) // SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid WHERE i0.a = 10
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT COUNT(*) FROM %j where %j._id in (select distinct parentid from %k) and %j.a = 10`, c, c, c, c)) // SELECT COUNT(*) FROM %j INNER JOIN %k ON %j._id = %k.parentid WHERE %j.a = 10
if err != nil {
t.Fatal(err)
}
@ -1349,7 +1349,7 @@ func TestPlanner_In(t *testing.T) {
})
t.Run("CountWithParentAndChildCondition", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 where i0._id in (select distinct parentid from i1 where x = 200) and i0.a = 10`) // SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid WHERE i0.a = 10
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT COUNT(*) FROM %j where %j._id in (select distinct parentid from %k where x = 200) and %j.a = 10`, c, c, c, c)) // SELECT COUNT(*) FROM %j INNER JOIN %k ON %j._id = %k.parentid WHERE %j.a = 10
if err != nil {
t.Fatal(err)
}
@ -1372,7 +1372,7 @@ func TestPlanner_Distinct(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
i0, err := c.GetHolder(0).CreateIndex(c.Idx("j"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -1381,7 +1381,7 @@ func TestPlanner_Distinct(t *testing.T) {
t.Fatal(err)
}
i1, err := c.GetHolder(0).CreateIndex("i1", pilosa.IndexOptions{TrackExistence: true})
i1, err := c.GetHolder(0).CreateIndex(c.Idx("k"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -1394,7 +1394,7 @@ func TestPlanner_Distinct(t *testing.T) {
// Populate with data.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i0",
Index: c.Idx("j"),
Query: `
Set(1, a=10)
Set(2, a=20)
@ -1404,7 +1404,7 @@ func TestPlanner_Distinct(t *testing.T) {
}
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i1",
Index: c.Idx("k"),
Query: `
Set(1, parentid=1)
Set(1, x=100)
@ -1419,7 +1419,7 @@ func TestPlanner_Distinct(t *testing.T) {
}
t.Run("SelectDistinct_id", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT distinct _id from i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT distinct _id from %j`, c))
if err != nil {
t.Fatal(err)
}
@ -1441,7 +1441,7 @@ func TestPlanner_Distinct(t *testing.T) {
t.Run("SelectDistinctNonId", func(t *testing.T) {
t.Skip("WIP")
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SELECT distinct parentid from i1`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`SELECT distinct parentid from %k`, c))
if err != nil {
t.Fatal(err)
}
@ -1462,7 +1462,7 @@ func TestPlanner_Distinct(t *testing.T) {
})
t.Run("SelectDistinctMultiple", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select distinct _id, parentid from i1`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`select distinct _id, parentid from %k`, c))
if err != nil {
t.Fatal(err)
}
@ -1488,7 +1488,7 @@ func TestPlanner_SelectTop(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
i0, err := c.GetHolder(0).CreateIndex(c.Idx("j"), pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -1503,7 +1503,7 @@ func TestPlanner_SelectTop(t *testing.T) {
// Populate with data.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i0",
Index: c.Idx("j"),
Query: `
Set(1, a=10)
Set(2, a=20)
@ -1516,7 +1516,7 @@ func TestPlanner_SelectTop(t *testing.T) {
}
t.Run("SelectTopStar", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select top(1) * from i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`select top(1) * from %j`, c))
if err != nil {
t.Fatal(err)
}
@ -1537,7 +1537,7 @@ func TestPlanner_SelectTop(t *testing.T) {
})
t.Run("SelectTopNStar", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select topn(1) * from i0`)
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, fmt.Sprintf(`select topn(1) * from %j`, c))
if err != nil {
t.Fatal(err)
}

7
stats/stats_test.go
View file

@ -112,7 +112,9 @@ func TestStatsCount_TopN(t *testing.T) {
}
func TestStatsCount_Bitmap(t *testing.T) {
c := test.MustRunCluster(t, 1)
// Cluster has to be unhsared because we're mocking the stats which writes
// to a holder in use by other tests.
c := test.MustRunUnsharedCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
@ -141,7 +143,8 @@ func TestStatsCount_Bitmap(t *testing.T) {
}
func TestStatsCount_APICalls(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
// We can't share a cluster when we're modifying its stats counter.
cluster := test.MustRunUnsharedCluster(t, 1)
defer cluster.Close()
cmd := cluster.GetNode(0)
h := cmd.Handler.(*pilosa.Handler).Handler

321
test/cluster.go
View file

@ -8,8 +8,10 @@ import (
"math"
"sort"
"strings"
"sync"
"testing"
"time"
"unicode"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/api/client"
@ -23,22 +25,86 @@ import (
"golang.org/x/sync/errgroup"
)
// modHasher represents a simple, mod-based hashing.
type ModHasher struct{}
func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n }
func (*ModHasher) Name() string { return "mod" }
// Cluster represents a Pilosa cluster (multiple Command instances)
// Cluster represents a per-test wrapper of a "real" cluster.
// Individual tests which request a cluster can get one of these
// back, wrapped with their test name.
type Cluster struct {
Nodes []*Command
tb testing.TB
*ShareableCluster
tb testing.TB
indexName string
indexBytes []byte
}
// Idx produces an index name suitable for this test's
// cluster, using the name itself, or the concatenation
// of that name and any provided strings, joined by underscores.
func (c *Cluster) Idx(of ...string) string {
if len(of) == 0 {
return c.indexName
}
for i := range of {
of[i] = indexName(of[i])
}
return c.indexName + strings.Join(of, "_")
}
// Format implements Formatter, allowing us to use clusters in
// format strings to handle the extremely common problem of "I
// want to embed the index name in this here string". Because we
// are horrible criminals, we append the rune, so you can use
// %i and %j to get index-name-plus-i and index-name-plus-j,
// respectively. We do assume that the rune works as a plain byte,
// though.
func (c *Cluster) Format(state fmt.State, r rune) {
c.indexBytes = append(c.indexBytes[:0], c.indexName...)
c.indexBytes = append(c.indexBytes, byte(r))
_, _ = state.Write(c.indexBytes)
}
func (c *Cluster) Close() error {
if c.shared {
// We're not going to actually close the cluster, BUT, we do want to
// delete and close all the indexes we might have just made.
h := c.GetHolder(0)
indexes := h.Indexes()
var lastErr error
api := c.Nodes[0].API
for _, idx := range indexes {
name := idx.Name()
if strings.HasPrefix(name, c.indexName) {
err := api.DeleteIndex(context.Background(), name)
if err != nil {
lastErr = err
}
}
}
return lastErr
}
if !c.started {
return errors.New("cluster not started yet")
}
return c.ShareableCluster.Close()
}
func (c *Cluster) Start() error {
if c.started {
return errors.New("cluster already started")
}
c.started = true
return c.ShareableCluster.Start()
}
// ShareableCluster represents a featurebase cluster (multiple Command instances)
// without test-specific overhead.
type ShareableCluster struct {
Nodes []*Command
started bool
shared bool
}
// Query executes an API.Query through one of the cluster's node's API. It fails
// the test if there is an error.
func (c *Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse {
func (c *ShareableCluster) Query(t testing.TB, index, query string) pilosa.QueryResponse {
t.Helper()
if len(c.Nodes) == 0 {
t.Fatal("must have at least one node in cluster to query")
@ -50,7 +116,7 @@ func (c *Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse
// QueryHTTP executes a PQL query through the HTTP endpoint. It fails
// the test for explicit errors, but returns an error which has the
// response body if the HTTP call returns a non-OK status.
func (c *Cluster) QueryHTTP(t testing.TB, index, query string) (string, error) {
func (c *ShareableCluster) QueryHTTP(t testing.TB, index, query string) (string, error) {
t.Helper()
if len(c.Nodes) == 0 {
t.Fatal("must have at least one node in cluster to query")
@ -61,7 +127,7 @@ func (c *Cluster) QueryHTTP(t testing.TB, index, query string) (string, error) {
// QueryGRPC executes a PQL query through the GRPC endpoint. It fails the
// test if there is an error.
func (c *Cluster) QueryGRPC(t testing.TB, index, query string) *proto.TableResponse {
func (c *ShareableCluster) QueryGRPC(t testing.TB, index, query string) *proto.TableResponse {
t.Helper()
if len(c.Nodes) == 0 {
t.Fatal("must have at least one node in cluster to query")
@ -93,7 +159,7 @@ func (c *Cluster) QueryGRPC(t testing.TB, index, query string) *proto.TableRespo
// starting; it's ok to reference each node by its index in the pre-sorted node
// list. It's also safe to use this method after `MustRunCluster()` if the
// cluster contains only one node.
func (c *Cluster) GetIdleNode(n int) *Command {
func (c *ShareableCluster) GetIdleNode(n int) *Command {
return c.Nodes[n]
}
@ -105,7 +171,7 @@ func (c *Cluster) GetIdleNode(n int) *Command {
// `123`, `789`, then we actually want `GetNode(0)` to return `c.Nodes[1]`, and
// `GetNode(1)` to return `c.Nodes[0]`. This method looks at all the node IDs,
// sorts them, and then returns the node that the test expects.
func (c *Cluster) GetNode(n int) *Command {
func (c *ShareableCluster) GetNode(n int) *Command {
// Put all the node IDs into a list to be sorted.
ids := make([]nodePlace, len(c.Nodes))
for i := range c.Nodes {
@ -126,7 +192,7 @@ func (c *Cluster) GetNode(n int) *Command {
// This used to be node0 in tests, but since implementing etcd, the primary
// can be any node in the cluster, so we have to use this method in tests which
// need to act on the primary.
func (c *Cluster) GetPrimary() *Command {
func (c *ShareableCluster) GetPrimary() *Command {
for _, n := range c.Nodes {
if n.IsPrimary() {
return n
@ -136,7 +202,7 @@ func (c *Cluster) GetPrimary() *Command {
}
// GetNonPrimary gets first first non-primary node in the list of nodes.
func (c *Cluster) GetNonPrimary() *Command {
func (c *ShareableCluster) GetNonPrimary() *Command {
for _, n := range c.Nodes {
if !n.IsPrimary() {
return n
@ -146,7 +212,7 @@ func (c *Cluster) GetNonPrimary() *Command {
}
// GetNonPrimaries gets all nodes except the primary.
func (c *Cluster) GetNonPrimaries() []*Command {
func (c *ShareableCluster) GetNonPrimaries() []*Command {
rtn := make([]*Command, 0)
for _, n := range c.Nodes {
if !n.IsPrimary() {
@ -162,15 +228,15 @@ type nodePlace struct {
idx int
}
func (c *Cluster) GetHolder(n int) *Holder {
func (c *ShareableCluster) GetHolder(n int) *Holder {
return &Holder{Holder: c.GetNode(n).Server.Holder()}
}
func (c *Cluster) Len() int {
func (c *ShareableCluster) Len() int {
return len(c.Nodes)
}
func (c *Cluster) ImportBitsWithTimestamp(t testing.TB, index, field string, rowcols [][2]uint64, timestamps []int64) {
func (c *ShareableCluster) ImportBitsWithTimestamp(t testing.TB, index, field string, rowcols [][2]uint64, timestamps []int64) {
t.Helper()
byShard := make(map[uint64][][2]uint64)
byShardTs := make(map[uint64][]int64)
@ -237,14 +303,15 @@ func (c *Cluster) ImportBitsWithTimestamp(t testing.TB, index, field string, row
}
}
}
func (c *Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint64) {
func (c *ShareableCluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint64) {
var noTime []int64
c.ImportBitsWithTimestamp(t, index, field, rowcols, noTime)
}
// ImportKeyKey imports data into an index where both the index and
// the field are using string keys.
func (c *Cluster) ImportKeyKey(t testing.TB, index, field string, valAndRecKeys [][2]string) {
func (c *ShareableCluster) ImportKeyKey(t testing.TB, index, field string, valAndRecKeys [][2]string) {
t.Helper()
importRequest := &pilosa.ImportRequest{
Index: index,
@ -273,7 +340,7 @@ type TimeQuantumKey struct {
// ImportTimeQuantumKey imports data into an index where the index is keyd
// and the field is a time-quantum
func (c *Cluster) ImportTimeQuantumKey(t testing.TB, index, field string, entries []TimeQuantumKey) {
func (c *ShareableCluster) ImportTimeQuantumKey(t testing.TB, index, field string, entries []TimeQuantumKey) {
t.Helper()
importRequest := &pilosa.ImportRequest{
Index: index,
@ -303,7 +370,7 @@ type IntKey struct {
}
// ImportIntKey imports int data into an index which uses string keys.
func (c *Cluster) ImportIntKey(t testing.TB, index, field string, pairs []IntKey) {
func (c *ShareableCluster) ImportIntKey(t testing.TB, index, field string, pairs []IntKey) {
t.Helper()
importRequest := &pilosa.ImportValueRequest{
Index: index,
@ -329,7 +396,7 @@ type IntID struct {
}
// ImportIntID imports data into an int field in an unkeyed index.
func (c *Cluster) ImportIntID(t testing.TB, index, field string, pairs []IntID) {
func (c *ShareableCluster) ImportIntID(t testing.TB, index, field string, pairs []IntID) {
t.Helper()
importRequest := &pilosa.ImportValueRequest{
Index: index,
@ -357,7 +424,7 @@ type KeyID struct {
}
//ImportIDKey imports data into an unkeyed set field in a keyed index.
func (c *Cluster) ImportIDKey(t testing.TB, index, field string, pairs []KeyID) {
func (c *ShareableCluster) ImportIDKey(t testing.TB, index, field string, pairs []KeyID) {
t.Helper()
importRequest := &pilosa.ImportRequest{
Index: index,
@ -378,7 +445,7 @@ func (c *Cluster) ImportIDKey(t testing.TB, index, field string, pairs []KeyID)
}
// CreateField creates the index (if necessary) and field specified.
func (c *Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field {
func (c *ShareableCluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field {
t.Helper()
idx, err := c.GetPrimary().API.CreateIndex(context.Background(), index, iopts)
if err != nil && !strings.Contains(err.Error(), "index already exists") {
@ -403,11 +470,7 @@ func (c *Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOpti
}
// Start runs a Cluster
func (c *Cluster) Start() error {
err := GetPortsGenConfigs(c.tb, c.Nodes)
if err != nil {
return errors.Wrap(err, "configuring cluster ports")
}
func (c *ShareableCluster) Start() error {
var eg errgroup.Group
for _, cc := range c.Nodes {
cc := cc
@ -415,7 +478,7 @@ func (c *Cluster) Start() error {
return cc.Start()
})
}
err = eg.Wait()
err := eg.Wait()
if err != nil {
return errors.Wrap(err, "starting cluster")
}
@ -423,7 +486,17 @@ func (c *Cluster) Start() error {
}
// Close stops a Cluster
func (c *Cluster) Close() error {
func (c *ShareableCluster) Close() error {
if c.shared {
for i := range c.Nodes {
holder := c.GetHolder(i)
indexes := holder.Indexes()
var indexNames []string
for _, idx := range indexes {
indexNames = append(indexNames, idx.Name())
}
}
}
for i, cc := range c.Nodes {
if err := cc.Close(); err != nil {
return errors.Wrapf(err, "stopping server %d", i)
@ -432,7 +505,10 @@ func (c *Cluster) Close() error {
return nil
}
func (c *Cluster) CloseAndRemoveNonPrimary() error {
func (c *ShareableCluster) CloseAndRemoveNonPrimary() error {
if c.shared {
return errors.New("can't close-and-remove in shared cluster")
}
for i, n := range c.Nodes {
if !n.IsPrimary() {
return c.CloseAndRemove(i)
@ -441,7 +517,10 @@ func (c *Cluster) CloseAndRemoveNonPrimary() error {
return errors.New("could not find non-primary node")
}
func (c *Cluster) CloseAndRemove(n int) error {
func (c *ShareableCluster) CloseAndRemove(n int) error {
if c.shared {
return errors.New("can't close-and-remove in shared cluster")
}
if n < 0 || n >= len(c.Nodes) {
return fmt.Errorf("close/remove from cluster: index %d out of range (len %d)", n, len(c.Nodes))
}
@ -455,7 +534,7 @@ func (c *Cluster) CloseAndRemove(n int) error {
// When this happens, we know etcd reached a combination of node states that
// would imply this cluster state, but some nodes may not have caught up yet;
// we just test that the coordinator thought the cluster was in the given state.
func (c *Cluster) AwaitPrimaryState(expectedState disco.ClusterState, timeout time.Duration) error {
func (c *ShareableCluster) AwaitPrimaryState(expectedState disco.ClusterState, timeout time.Duration) error {
if len(c.Nodes) < 1 {
return errors.New("can't await coordinator state on an empty cluster")
}
@ -474,16 +553,15 @@ func (c *Cluster) AwaitPrimaryState(expectedState disco.ClusterState, timeout ti
return errors.New("timed out waiting for cluster to have valid topology")
}
// we used up some of our timeout waiting for this
c.tb.Logf("had to wait %v for cluster topology", elapsed)
timeout -= elapsed
}
onlyCoordinator := &Cluster{Nodes: []*Command{primary}}
onlyCoordinator := &ShareableCluster{Nodes: []*Command{primary}}
return onlyCoordinator.AwaitState(expectedState, timeout)
}
// ExceptionalState returns an error if any node in the cluster is not
// in the expected state.
func (c *Cluster) ExceptionalState(expectedState disco.ClusterState) error {
func (c *ShareableCluster) ExceptionalState(expectedState disco.ClusterState) error {
for _, node := range c.Nodes {
state, err := node.API.State()
if err != nil || state != expectedState {
@ -494,7 +572,7 @@ func (c *Cluster) ExceptionalState(expectedState disco.ClusterState) error {
}
// AwaitState waits for the whole cluster to reach a specified state.
func (c *Cluster) AwaitState(expectedState disco.ClusterState, timeout time.Duration) (err error) {
func (c *ShareableCluster) AwaitState(expectedState disco.ClusterState, timeout time.Duration) (err error) {
if len(c.Nodes) < 1 {
return errors.New("can't await state of an empty cluster")
}
@ -512,7 +590,8 @@ func (c *Cluster) AwaitState(expectedState disco.ClusterState, timeout time.Dura
elapsed, expectedState, err)
}
// MustNewCluster creates a new cluster. If opts contains only one
// MustNewCluster creates a new cluster or returns an existing one. It never shares
// a cluster with non-empty opts. If opts contains only one
// slice of command options, those options are used with every node.
// If it is empty, default options are used. Otherwise, it must contain size
// slices of command options, which are used with corresponding nodes.
@ -522,29 +601,111 @@ func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) *Cl
}
tb.Helper()
shareable := len(opts) == 0
// We want tests to default to using the in-memory translate store, so we
// prepend opts with that functional option. If a different translate store
// has been specified, it will override this one.
opts = prependOpts(opts, size)
c, err := newCluster(tb, size, opts...)
c, err := newCluster(tb, size, shareable, opts...)
if err != nil {
tb.Fatalf("new cluster: %v", err)
}
return c
}
// newCluster creates a new cluster
func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (*Cluster, error) {
if size == 0 {
return nil, errors.New("cluster must contain at least one node")
// MustUnsharedCluster creates a new cluster. If opts contains only one
// slice of command options, those options are used with every node.
// If it is empty, default options are used. Otherwise, it must contain size
// slices of command options, which are used with corresponding nodes. The
// new cluster is always unshared.
func MustUnsharedCluster(tb testing.TB, size int, opts ...[]server.CommandOption) *Cluster {
if size > 1 && !etcd.AllowCluster() {
tb.Skip("Testing PLG which does not allow clustering")
}
tb.Helper()
// We want tests to default to using the in-memory translate store, so we
// prepend opts with that functional option. If a different translate store
// has been specified, it will override this one.
opts = prependOpts(opts, size)
if len(opts) != size && len(opts) != 0 && len(opts) != 1 {
return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes")
c, err := newCluster(tb, size, false, opts...)
if err != nil {
tb.Fatalf("new cluster: %v", err)
}
return c
}
cluster := &Cluster{Nodes: make([]*Command, size), tb: tb}
// MustRunUnsharedCluster creates a new cluster. If opts contains only one
// slice of command options, those options are used with every node.
// If it is empty, default options are used. Otherwise, it must contain size
// slices of command options, which are used with corresponding nodes. The
// new cluster is always unshared. The new cluster is started automatically,
// or the test is failed.
func MustRunUnsharedCluster(tb testing.TB, size int, opts ...[]server.CommandOption) *Cluster {
if size > 1 && !etcd.AllowCluster() {
tb.Skip("Testing PLG which does not allow clustering")
}
tb.Helper()
// We want tests to default to using the in-memory translate store, so we
// prepend opts with that functional option. If a different translate store
// has been specified, it will override this one.
opts = prependOpts(opts, size)
c, err := newCluster(tb, size, false, opts...)
if err != nil {
tb.Fatalf("new cluster: %v", err)
}
err = c.Start()
if err != nil {
tb.Fatalf("starting cluster: %v", err)
}
return c
}
type clusterCache struct {
mu sync.Mutex
clusters map[int]*ShareableCluster
}
// CleanupClusters calls the close functions on any shared clusters that are
// still open.
func (c *clusterCache) CleanupClusters() {
for k, v := range c.clusters {
_ = v.Close()
delete(c.clusters, k)
}
}
func (c *clusterCache) newCluster(tb testing.TB, size int) (*ShareableCluster, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c := c.clusters[size]; c != nil {
return c, nil
}
// Make a whole-test wrapper so that the cluster we create will use the provided tb
// for nearly everything, but the call to TempDir inside NewCommand will pick up a
// persistent directory which outlives the provided TB.
newTB := NewWholeTestRun(tb)
if c.clusters == nil {
c.clusters = make(map[int]*ShareableCluster)
// tb should always be a wholeTestRun for clusterCache, and we need to
// register with that, so our cleanup happens *before* the deletion of
// the directories, otherwise etcd can fail to flush WAL files on
// exit, causing tests to fail.
newTB.Cleanup(c.CleanupClusters)
}
cluster, err := underlyingNewCluster(newTB, size)
if err != nil {
return nil, err
}
cluster.shared = true
c.clusters[size] = cluster
return cluster, nil
}
func underlyingNewCluster(tb DirCleaner, size int, opts ...[]server.CommandOption) (*ShareableCluster, error) {
cluster := &ShareableCluster{Nodes: make([]*Command, size)}
for i := 0; i < size; i++ {
var commandOpts []server.CommandOption
if len(opts) > 0 {
@ -555,17 +716,51 @@ func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (*Clust
cluster.Nodes[i] = m
}
// The GetPorts... stuff calls things elsewhere that want a plain testing.TB,
// and doesn't produce permanent directories, I think.
err := GetPortsGenConfigs(tb, cluster.Nodes)
if err != nil {
return nil, errors.Wrap(err, "configuring cluster ports")
}
return cluster, nil
}
var globalClusterCache clusterCache
// newCluster creates a new cluster, using the shared cluster cache if no opts are
// specified.
func newCluster(tb testing.TB, size int, shareable bool, opts ...[]server.CommandOption) (*Cluster, error) {
if size == 0 {
return nil, errors.New("cluster must contain at least one node")
}
if len(opts) != size && len(opts) != 0 && len(opts) != 1 {
return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes")
}
var shared *ShareableCluster
var err error
if !shareable {
shared, err = underlyingNewCluster(tb, size, opts...)
} else {
shared, err = globalClusterCache.newCluster(tb, size)
}
if err != nil {
return nil, err
}
return &Cluster{ShareableCluster: shared, tb: tb, indexName: indexName(tb.Name())}, nil
}
// MustRunCluster creates and starts a new cluster. The opts parameter
// is slightly magical; see MustNewCluster.
func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) *Cluster {
cluster := MustNewCluster(tb, size, opts...)
err := cluster.Start()
if err != nil {
tb.Fatalf("run cluster: %v", err)
if !cluster.started {
err := cluster.Start()
if err != nil {
tb.Fatalf("run cluster: %v", err)
}
cluster.started = true
}
return cluster
}
@ -613,3 +808,19 @@ func prependTestServerOpts(opts []server.CommandOption) []server.CommandOption {
}
return append(defaultOpts, opts...)
}
func indexName(in string) string {
return strings.Map(func(r rune) rune {
if r < 127 {
switch {
case unicode.IsLetter(r):
return unicode.ToLower(r)
case unicode.IsNumber(r):
return r
case r == '/':
return '_'
}
}
return -1
}, in)
}

3
test/disco.go
View file

@ -6,7 +6,6 @@ import (
"fmt"
"net"
"strings"
"testing"
"github.com/featurebasedb/featurebase/v3/etcd"
"github.com/featurebasedb/featurebase/v3/server"
@ -56,7 +55,7 @@ func listenerWithURL() (listener *net.TCPListener, url string, err error) {
// and modifies the configs of the provided Command objects
// to point to these etcd configs. It uses etcd.GenEtcdConfigs,
// which in turn creates temporary directories and the like.
func GetPortsGenConfigs(tb testing.TB, nodes []*Command) error {
func GetPortsGenConfigs(tb DirCleaner, nodes []*Command) error {
clusterName, cfgs := etcd.GenEtcdConfigs(tb, len(nodes))
for i := range nodes {
if nodes[i].Config == nil {

129
test/glue.go Normal file
View file

@ -0,0 +1,129 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package test
import (
"fmt"
"log"
"os"
"sync"
"testing"
"github.com/molecula/featurebase/v3/testhook"
)
// glue lets us make a thing which isn't a testing.TB, but can be passed
// around to the functions that take testing.TB.
// DirCleaner represents the subset of the testing.TB interface
// we care about, allowing us to take objects which behave like
// that without importing all of testing to get them.
type DirCleaner interface {
Helper()
Errorf(string, ...interface{})
Fatalf(string, ...interface{})
Fatal(...interface{})
Logf(string, ...interface{})
Name() string
TempDir() string
Cleanup(func())
Skip(...interface{})
}
// Verify that a TB is a DirCleaner
var _ DirCleaner = testing.TB(nil)
var _ DirCleaner = &wholeTestRunWrapper{}
// wholeTestRun is a thing that's shaped a bit like testing.TB,
// but it can be used across all the tests, running its cleanup functions
// at the very end of the testing process. this lets us create clusters
// using the same code and logic we would for per-test things, except
// substituting this, and then have a single global post-test-hook run
// their cleanup.
type wholeTestRun struct {
setup sync.Once
mu sync.Mutex
tempDirs []string
cleanupFuncs []func()
}
// wholeTestRunWrapper is a test-specific thing that can refer to the
// global shared state, but also forwards everything *except* TempDir,
// Cleanup, and Logf to the tb it's created with.
type wholeTestRunWrapper struct {
testing.TB
}
// globalT is a system-wide wholeTestRun. the first time it's used, for any
// reason, it registers a cleanup with testhook, which will run at the end
// of the TestMain stuff, but before any previously-registered cleanup
// functions (such as the auditor stuff) so it can ensure that everything's
// been deleted. Basically this exists to let us call `tb.TempDir` on
// things and get a directory which outlives the current test.
var globalT wholeTestRun
// NewWholeTestRun produces a wholeTestRunWrapper around TB, which overrides
// a couple of the TB's methods to get whole-test-friendly behaviors.
func NewWholeTestRun(tb testing.TB) *wholeTestRunWrapper {
return &wholeTestRunWrapper{TB: tb}
}
func (w *wholeTestRun) Teardown() {
w.mu.Lock()
defer w.mu.Unlock()
for _, fn := range w.cleanupFuncs {
fn()
}
fmt.Fprintf(os.Stderr, "tearing down: %d directories\n", len(w.tempDirs))
for _, path := range w.tempDirs {
fmt.Fprintf(os.Stderr, "tearing down: deleting %s\n", path)
// disregard errors because we don't care that much about them;
// we assume RemoveAll probably works unless something's wrong.
// see the comments on the testing package's internal removeAll,
// which suggests the errors only happen on Windows, which we
// don't support.
_ = os.RemoveAll(path)
}
}
func (w *wholeTestRun) Setup() {
w.setup.Do(func() {
testhook.RegisterPostTestHook(func() error {
w.Teardown()
return nil
})
})
}
// We provide a Logf here because the wholeTestRunWrapper could
// be set as a tb-replacement for something which lasts past the
// test that created it. That shouldn't happen, probably.
func (w *wholeTestRunWrapper) Logf(msg string, args ...interface{}) {
log.Printf(msg, args...)
}
func (w *wholeTestRun) TempDir(tb DirCleaner) string {
w.Setup()
w.mu.Lock()
defer w.mu.Unlock()
path, err := os.MkdirTemp("", "test-temp-")
if err != nil {
tb.Fatalf("creating temp dir: %v", err)
}
globalT.tempDirs = append(globalT.tempDirs, path)
return path
}
func (w *wholeTestRun) Cleanup(fn func()) {
w.Setup()
w.mu.Lock()
defer w.mu.Unlock()
w.cleanupFuncs = append(w.cleanupFuncs, fn)
}
func (w *wholeTestRunWrapper) TempDir() string {
return globalT.TempDir(w.TB)
}
func (w *wholeTestRunWrapper) Cleanup(fn func()) {
globalT.Cleanup(fn)
}

22
test/pilosa.go
View file

@ -14,11 +14,10 @@ import (
"testing"
"time"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/encoding/proto"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/testhook"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/server"
)
// //////////////////////////////////////////////////////////////////////////////////
@ -37,11 +36,8 @@ func OptAllowedOrigins(origins []string) server.CommandOption {
}
// newCommand returns a new instance of Main with a temporary data directory and random port.
func newCommand(tb testing.TB, opts ...server.CommandOption) *Command {
path, err := testhook.TempDir(tb, "pilosa-command-")
if err != nil {
panic(err)
}
func newCommand(tb DirCleaner, opts ...server.CommandOption) *Command {
path := tb.TempDir()
// Set aggressive close timeout by default to avoid hanging tests. This was
// a problem with PDK tests which used pilosa/client as well. We put it at the
@ -54,7 +50,7 @@ func newCommand(tb testing.TB, opts ...server.CommandOption) *Command {
m := &Command{commandOptions: opts}
m.Command = server.NewCommand(bytes.NewReader(nil), io.Discard, io.Discard, opts...)
// pick etcd ports using a socket rather than a real port
err = GetPortsGenConfigs(tb, []*Command{m})
err := GetPortsGenConfigs(tb, []*Command{m})
if err != nil {
tb.Fatalf("generating config: %v", err)
}
@ -81,7 +77,7 @@ func newCommand(tb testing.TB, opts ...server.CommandOption) *Command {
}
// NewCommandNode returns a new instance of Command with clustering enabled.
func NewCommandNode(tb testing.TB, opts ...server.CommandOption) *Command {
func NewCommandNode(tb DirCleaner, opts ...server.CommandOption) *Command {
// We want tests to default to using the in-memory translate store, so we
// prepend opts with that functional option. If a different translate store
// has been specified, it will override this one.
@ -96,7 +92,7 @@ func RunCommand(t *testing.T) *Command {
// prefer MustRunCluster since it sets up for using etcd using
// the GenDisCoConfig(size) option.
return MustRunCluster(t, 1).GetNode(0)
return MustRunUnsharedCluster(t, 1).GetNode(0)
}
// Close closes the program and removes the underlying data directory.

6
testdata/schema.json vendored
View file

@ -1,7 +1,7 @@
{
"indexes": [
{
"name": "equipment",
"name": "e",
"options": {
"keys": true,
"trackExistence": true
@ -137,7 +137,7 @@
"shardWidth": 1048576
},
{
"name": "power_ts",
"name": "t",
"options": {
"keys": true,
"trackExistence": true
@ -285,7 +285,7 @@
"shardWidth": 1048576
},
{
"name": "sites",
"name": "s",
"options": {
"keys": false,
"trackExistence": true

152
translator_test.go
View file

@ -12,12 +12,10 @@ import (
"time"
"github.com/google/go-cmp/cmp"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/boltdb"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/mock"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/test"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/mock"
"github.com/molecula/featurebase/v3/test"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
@ -151,32 +149,7 @@ func TestMultiTranslateEntryReader(t *testing.T) {
}
func TestTranslation_KeyNotFound(t *testing.T) {
c := test.MustRunCluster(t, 4,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node3"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(t, 4)
defer c.Close()
node0 := c.GetNode(0)
@ -185,19 +158,19 @@ func TestTranslation_KeyNotFound(t *testing.T) {
node3 := c.GetNode(3)
ctx := context.Background()
idx, fld := "i", "f"
index, fld := c.Idx(), "f"
// Create an index with keys.
if _, err := node0.API.CreateIndex(ctx, idx, pilosa.IndexOptions{Keys: true}); err != nil {
if _, err := node0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true}); err != nil {
t.Fatal(err)
}
// Create an index with keys.
if _, err := node0.API.CreateField(ctx, idx, fld, pilosa.OptFieldKeys()); err != nil {
if _, err := node0.API.CreateField(ctx, index, fld, pilosa.OptFieldKeys()); err != nil {
t.Fatal(err)
}
// write a new key and get id
req, err := node0.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
Index: idx,
Index: index,
Field: fld,
Keys: []string{"k1"},
NotWritable: false,
@ -217,7 +190,7 @@ func TestTranslation_KeyNotFound(t *testing.T) {
// read non-existing key
req, err = node3.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
Index: idx,
Index: index,
Field: fld,
Keys: []string{"k2"},
NotWritable: true,
@ -235,7 +208,7 @@ func TestTranslation_KeyNotFound(t *testing.T) {
}
req, err = node1.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
Index: idx,
Index: index,
Keys: []string{"k2"},
NotWritable: true,
})
@ -252,7 +225,7 @@ func TestTranslation_KeyNotFound(t *testing.T) {
}
req, err = node2.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
Index: idx,
Index: index,
Field: fld,
Keys: []string{"k2", "k1"},
NotWritable: false,
@ -307,37 +280,18 @@ func TestTranslation_Primary(t *testing.T) {
// non-primary nodes are forwarded to the primary.
t.Run("ForwardFieldKey", func(t *testing.T) {
// Start a 2-node cluster.
c := test.MustRunCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(t, 3)
defer c.Close()
node0 := c.GetPrimary()
node1 := c.GetNonPrimary()
ctx := context.Background()
idx := "i"
index := c.Idx()
fld := "f"
// Create an index without keys.
if _, err := node1.API.CreateIndex(ctx, idx,
if _, err := node1.API.CreateIndex(ctx, index,
pilosa.IndexOptions{
Keys: false,
}); err != nil {
@ -345,7 +299,7 @@ func TestTranslation_Primary(t *testing.T) {
}
// Create a field with keys.
if _, err := node1.API.CreateField(ctx, idx, fld,
if _, err := node1.API.CreateField(ctx, index, fld,
pilosa.OptFieldKeys(),
); err != nil {
t.Fatal(err)
@ -357,7 +311,7 @@ func TestTranslation_Primary(t *testing.T) {
// Send a translation request to node1 (non-primary).
_, err := node1.API.Query(ctx,
&pilosa.QueryRequest{Index: idx, Query: pql},
&pilosa.QueryRequest{Index: index, Query: pql},
)
if err != nil {
t.Fatal(err)
@ -368,7 +322,7 @@ func TestTranslation_Primary(t *testing.T) {
// Read the row and ensure the key was set.
qry := fmt.Sprintf(`Row(%s="%s")`, fld, keys[i])
resp, err := node0.API.Query(ctx,
&pilosa.QueryRequest{Index: idx, Query: qry},
&pilosa.QueryRequest{Index: index, Query: qry},
)
if err != nil {
t.Fatal(err)
@ -383,53 +337,28 @@ func TestTranslation_Primary(t *testing.T) {
}
func TestTranslation_TranslateIDsOnCluster(t *testing.T) {
c := test.MustRunCluster(t, 4,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node3"),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(t, 4)
defer c.Close()
coord := c.GetPrimary()
other := c.GetNonPrimary()
ctx := context.Background()
idx, fld := "i", "f"
index, fld := c.Idx(), "f"
// Create an index with keys.
if _, err := coord.API.CreateIndex(ctx, idx, pilosa.IndexOptions{Keys: true}); err != nil {
if _, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true}); err != nil {
t.Fatal(err)
}
// Create an index with keys.
if _, err := coord.API.CreateField(ctx, idx, fld, pilosa.OptFieldKeys()); err != nil {
if _, err := coord.API.CreateField(ctx, index, fld, pilosa.OptFieldKeys()); err != nil {
t.Fatal(err)
}
keys := []string{"k0", "k1", "k2", "k3", "k4", "k5", "k6", "k7", "k8", "k9"}
// write a new key and get id
req, err := coord.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
Index: idx,
Index: index,
Field: fld,
Keys: keys,
NotWritable: false,
@ -451,7 +380,7 @@ func TestTranslation_TranslateIDsOnCluster(t *testing.T) {
// translate ids
req, err = other.API.Serializer.Marshal(&pilosa.TranslateIDsRequest{
Index: idx,
Index: index,
Field: fld,
IDs: ids,
})
@ -473,7 +402,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
c.CreateField(t, "i", pilosa.IndexOptions{Keys: true}, "f", pilosa.OptFieldKeys())
c.CreateField(t, c.Idx(), pilosa.IndexOptions{Keys: true}, "f", pilosa.OptFieldKeys())
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
@ -503,7 +432,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
for i, keys := range parts {
i, keys := i, keys
g.Go(func() error {
_, err := c.GetNode(i).API.CreateIndexKeys(ctx, "i", keys...)
_, err := c.GetNode(i).API.CreateIndexKeys(ctx, c.Idx(), keys...)
return err
})
}
@ -522,7 +451,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
}
// Obtain authoritative translations for the keys.
translations, err := c.GetPrimary().API.FindIndexKeys(ctx, "i", keyList...)
translations, err := c.GetPrimary().API.FindIndexKeys(ctx, c.Idx(), keyList...)
if err != nil {
t.Errorf("obtaining authoritative translations: %v", err)
return
@ -540,7 +469,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
x, api := i, n.API
g.Go(func() (err error) {
defer func() { err = errors.Wrapf(err, "translating on node %d", x) }()
localTranslations, err := api.FindIndexKeys(ctx, "i", keyList...)
localTranslations, err := api.FindIndexKeys(ctx, c.Idx(), keyList...)
if err != nil {
return errors.Wrap(err, "finding translations")
}
@ -557,7 +486,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
x, api := i, n.API
g.Go(func() (err error) {
defer func() { err = errors.Wrapf(err, "translating on node %d", x) }()
localTranslations, err := api.CreateIndexKeys(ctx, "i", keyList...)
localTranslations, err := api.CreateIndexKeys(ctx, c.Idx(), keyList...)
if err != nil {
return errors.Wrap(err, "finding translations")
}
@ -589,7 +518,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
for i, keys := range parts {
i, keys := i, keys
g.Go(func() error {
_, err := c.GetNode(i).API.CreateFieldKeys(ctx, "i", "f", keys...)
_, err := c.GetNode(i).API.CreateFieldKeys(ctx, c.Idx(), "f", keys...)
return err
})
}
@ -608,7 +537,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
}
// Obtain authoritative translations for the keys.
translations, err := c.GetPrimary().API.FindFieldKeys(ctx, "i", "f", keyList...)
translations, err := c.GetPrimary().API.FindFieldKeys(ctx, c.Idx(), "f", keyList...)
if err != nil {
t.Errorf("obtaining authoritative translations: %v", err)
return
@ -626,7 +555,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
x, api := i, n.API
g.Go(func() (err error) {
defer func() { err = errors.Wrapf(err, "translating on node %d", x) }()
localTranslations, err := api.FindFieldKeys(ctx, "i", "f", keyList...)
localTranslations, err := api.FindFieldKeys(ctx, c.Idx(), "f", keyList...)
if err != nil {
return errors.Wrap(err, "finding translations")
}
@ -643,7 +572,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
x, api := i, n.API
g.Go(func() (err error) {
defer func() { err = errors.Wrapf(err, "translating on node %d", x) }()
localTranslations, err := api.CreateFieldKeys(ctx, "i", "f", keyList...)
localTranslations, err := api.CreateFieldKeys(ctx, c.Idx(), "f", keyList...)
if err != nil {
return errors.Wrap(err, "finding translations")
}
@ -661,19 +590,20 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
func TestTranslation_Cluster_CreateFindUnkeyed(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
i := c.Idx()
c.CreateField(t, "i", pilosa.IndexOptions{}, "f")
c.CreateField(t, i, pilosa.IndexOptions{}, "f")
t.Run("Index", func(t *testing.T) {
t.Run("Create", func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := c.GetNonPrimary().API.CreateIndexKeys(ctx, "i", "foo")
_, err := c.GetNonPrimary().API.CreateIndexKeys(ctx, i, "foo")
if err == nil {
t.Fatal("unexpected success")
}
expect := `cannot create keys on unkeyed index "i"`
expect := fmt.Sprintf(`cannot create keys on unkeyed index "%s"`, i)
if got := err.Error(); got != expect {
t.Fatalf("expected error %q but got %q", expect, got)
}
@ -682,11 +612,11 @@ func TestTranslation_Cluster_CreateFindUnkeyed(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := c.GetNonPrimary().API.FindIndexKeys(ctx, "i", "foo")
_, err := c.GetNonPrimary().API.FindIndexKeys(ctx, i, "foo")
if err == nil {
t.Fatal("unexpected success")
}
expect := `cannot find keys on unkeyed index "i"`
expect := fmt.Sprintf(`cannot find keys on unkeyed index "%s"`, i)
if got := err.Error(); got != expect {
t.Fatalf("expected error %q but got %q", expect, got)
}
@ -697,7 +627,7 @@ func TestTranslation_Cluster_CreateFindUnkeyed(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := c.GetNonPrimary().API.CreateFieldKeys(ctx, "i", "f", "foo")
_, err := c.GetNonPrimary().API.CreateFieldKeys(ctx, i, "f", "foo")
if err == nil {
t.Fatal("unexpected success")
}
@ -710,7 +640,7 @@ func TestTranslation_Cluster_CreateFindUnkeyed(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := c.GetNonPrimary().API.FindFieldKeys(ctx, "i", "f", "foo")
_, err := c.GetNonPrimary().API.FindFieldKeys(ctx, i, "f", "foo")
if err == nil {
t.Fatal("unexpected success")
}

18
tx_test.go
View file

@ -7,10 +7,9 @@ import (
"fmt"
"testing"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/test"
. "github.com/featurebasedb/featurebase/v3/vprint" // nolint:staticcheck
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/test"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
)
func queryIRABit(m0api *pilosa.API, acctOwnerID uint64, iraField string, iraRowID uint64, index string) (bit bool) {
@ -46,21 +45,14 @@ func queryBalances(m0api *pilosa.API, acctOwnerID uint64, fldAcct0, fldAcct1, in
}
func TestAPI_ImportAtomicRecord(t *testing.T) {
c := test.MustRunCluster(t, 1,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(t, 1)
defer c.Close()
m0 := c.GetNode(0)
m0api := m0.API
ctx := context.Background()
index := "i"
index := c.Idx()
fieldAcct0 := "acct0"
fieldAcct1 := "acct1"

14
utils_internal_test.go
View file

@ -17,7 +17,7 @@ import (
// NewTestCluster returns a cluster with n nodes and uses a mod-based hasher.
func NewTestCluster(tb testing.TB, n int) *cluster {
if n > 1 && etcd.AllowCluster() {
if n > 1 && !etcd.AllowCluster() {
tb.Skipf("cluster size %d not supported in unclustered mode", n)
}
path, err := testhook.TempDir(tb, "pilosa-cluster-")
@ -28,7 +28,7 @@ func NewTestCluster(tb testing.TB, n int) *cluster {
availableShardFileFlushDuration.Set(100 * time.Millisecond)
c := newCluster()
c.ReplicaN = 1
c.Hasher = NewTestModHasher()
c.Hasher = &disco.Jmphasher{}
c.Path = path
nodes := make([]*disco.Node, 0, n)
@ -63,16 +63,6 @@ func NewTestURIFromHostPort(host string, port uint16) pnet.URI {
return *uri
}
// ModHasher represents a simple, mod-based hashing.
type TestModHasher struct{}
// NewTestModHasher returns a new instance of ModHasher with n buckets.
func NewTestModHasher() *TestModHasher { return &TestModHasher{} }
func (*TestModHasher) Hash(key uint64, n int) int { return int(key) % n }
func (*TestModHasher) Name() string { return "mod" }
func TestReplaceFirstFromBack(t *testing.T) {
for name, test := range map[string]struct {
input string