mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-10 07:01:01 +00:00
bc13834343
port mapper gives out ports from 63000-65000 for the tests fix another race http test uses port.MustGetPort rbf: remove :0 port request ocd happy test fix for grpc listener address already in use test/disco allocates BindGRPC port from the port mapper dump stack on each GetPort verify each port is usable right away server/config.go has Config.Validate() now panic if gossip port is 0. validate server.Config fix another gossip port 0 builds quiet, don't dump stack on each port alloc builds happy linter even gossip fallback should not be zero but rather use the port mapper
908 lines
26 KiB
Go
908 lines
26 KiB
Go
// Copyright 2017 Pilosa Corp.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package pilosa_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
"github.com/pilosa/pilosa/v2"
|
|
"github.com/pilosa/pilosa/v2/boltdb"
|
|
"github.com/pilosa/pilosa/v2/http"
|
|
"github.com/pilosa/pilosa/v2/mock"
|
|
"github.com/pilosa/pilosa/v2/server"
|
|
"github.com/pilosa/pilosa/v2/test"
|
|
"github.com/pilosa/pilosa/v2/topology"
|
|
"github.com/pkg/errors"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
func TestInMemTranslateStore_TranslateKey(t *testing.T) {
|
|
s := pilosa.NewInMemTranslateStore("IDX", "FLD", 0, topology.DefaultPartitionN)
|
|
|
|
// Ensure initial key translates to ID 1.
|
|
if id, err := s.TranslateKey("foo", true); err != nil {
|
|
t.Fatal(err)
|
|
} else if got, want := id, uint64(1); got != want {
|
|
t.Fatalf("TranslateKey()=%d, want %d", got, want)
|
|
}
|
|
|
|
// Ensure next key autoincrements.
|
|
if id, err := s.TranslateKey("bar", true); err != nil {
|
|
t.Fatal(err)
|
|
} else if got, want := id, uint64(2); got != want {
|
|
t.Fatalf("TranslateKey()=%d, want %d", got, want)
|
|
}
|
|
|
|
// Ensure retranslating existing key returns original ID.
|
|
if id, err := s.TranslateKey("foo", true); err != nil {
|
|
t.Fatal(err)
|
|
} else if got, want := id, uint64(1); got != want {
|
|
t.Fatalf("TranslateKey()=%d, want %d", got, want)
|
|
}
|
|
}
|
|
|
|
func TestInMemTranslateStore_TranslateID(t *testing.T) {
|
|
s := pilosa.NewInMemTranslateStore("IDX", "FLD", 0, topology.DefaultPartitionN)
|
|
|
|
// Setup initial keys.
|
|
if _, err := s.TranslateKey("foo", true); err != nil {
|
|
t.Fatal(err)
|
|
} else if _, err := s.TranslateKey("bar", true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Ensure IDs can be translated back to keys.
|
|
if key, err := s.TranslateID(1); err != nil {
|
|
t.Fatal(err)
|
|
} else if got, want := key, "foo"; got != want {
|
|
t.Fatalf("TranslateID()=%s, want %s", got, want)
|
|
}
|
|
|
|
if key, err := s.TranslateID(2); err != nil {
|
|
t.Fatal(err)
|
|
} else if got, want := key, "bar"; got != want {
|
|
t.Fatalf("TranslateID()=%s, want %s", got, want)
|
|
}
|
|
}
|
|
|
|
func TestMultiTranslateEntryReader(t *testing.T) {
|
|
t.Run("None", func(t *testing.T) {
|
|
r := pilosa.NewMultiTranslateEntryReader(context.Background(), nil)
|
|
defer r.Close()
|
|
|
|
var entry pilosa.TranslateEntry
|
|
if err := r.ReadEntry(&entry); err != io.EOF {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
t.Run("Single", func(t *testing.T) {
|
|
var r0 mock.TranslateEntryReader
|
|
r0.CloseFunc = func() error { return nil }
|
|
r0.ReadEntryFunc = func(entry *pilosa.TranslateEntry) error {
|
|
*entry = pilosa.TranslateEntry{Index: "i", Field: "f", ID: 1, Key: "foo"}
|
|
return nil
|
|
}
|
|
r := pilosa.NewMultiTranslateEntryReader(context.Background(), []pilosa.TranslateEntryReader{&r0})
|
|
defer r.Close()
|
|
|
|
var entry pilosa.TranslateEntry
|
|
if err := r.ReadEntry(&entry); err != nil {
|
|
t.Fatal(err)
|
|
} else if diff := cmp.Diff(entry, pilosa.TranslateEntry{Index: "i", Field: "f", ID: 1, Key: "foo"}); diff != "" {
|
|
t.Fatal(diff)
|
|
}
|
|
if err := r.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
t.Run("Multi", func(t *testing.T) {
|
|
ready0, ready1 := make(chan struct{}), make(chan struct{})
|
|
|
|
var r0 mock.TranslateEntryReader
|
|
r0.CloseFunc = func() error { return nil }
|
|
r0.ReadEntryFunc = func(entry *pilosa.TranslateEntry) error {
|
|
if _, ok := <-ready0; !ok {
|
|
return io.EOF
|
|
}
|
|
*entry = pilosa.TranslateEntry{Index: "i0", Field: "f0", ID: 1, Key: "foo"}
|
|
return nil
|
|
}
|
|
|
|
var r1 mock.TranslateEntryReader
|
|
r1.CloseFunc = func() error { return nil }
|
|
r1.ReadEntryFunc = func(entry *pilosa.TranslateEntry) error {
|
|
if _, ok := <-ready1; !ok {
|
|
return io.EOF
|
|
}
|
|
*entry = pilosa.TranslateEntry{Index: "i1", Field: "f1", ID: 2, Key: "bar"}
|
|
return nil
|
|
}
|
|
|
|
r := pilosa.NewMultiTranslateEntryReader(context.Background(), []pilosa.TranslateEntryReader{&r1, &r0})
|
|
defer r.Close()
|
|
|
|
// Ensure r0 is read first
|
|
ready0 <- struct{}{}
|
|
var entry pilosa.TranslateEntry
|
|
if err := r.ReadEntry(&entry); err != nil {
|
|
t.Fatal(err)
|
|
} else if diff := cmp.Diff(entry, pilosa.TranslateEntry{Index: "i0", Field: "f0", ID: 1, Key: "foo"}); diff != "" {
|
|
t.Fatal(diff)
|
|
}
|
|
|
|
// Unblock r1.
|
|
ready1 <- struct{}{}
|
|
|
|
// Read from r1 next.
|
|
if err := r.ReadEntry(&entry); err != nil {
|
|
t.Fatal(err)
|
|
} else if diff := cmp.Diff(entry, pilosa.TranslateEntry{Index: "i1", Field: "f1", ID: 2, Key: "bar"}); diff != "" {
|
|
t.Fatal(diff)
|
|
}
|
|
|
|
// Close both readers.
|
|
close(ready0)
|
|
close(ready1)
|
|
if err := r.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
t.Run("Error", func(t *testing.T) {
|
|
var r0 mock.TranslateEntryReader
|
|
r0.CloseFunc = func() error { return nil }
|
|
r0.ReadEntryFunc = func(entry *pilosa.TranslateEntry) error {
|
|
return errors.New("marker")
|
|
}
|
|
r := pilosa.NewMultiTranslateEntryReader(context.Background(), []pilosa.TranslateEntryReader{&r0})
|
|
defer r.Close()
|
|
|
|
var entry pilosa.TranslateEntry
|
|
if err := r.ReadEntry(&entry); err == nil || err.Error() != `marker` {
|
|
t.Fatalf("unexpected error: %s", err)
|
|
}
|
|
if err := r.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
}
|
|
|
|
// Test key translation with multiple nodes.
|
|
func TestTranslation_Reset(t *testing.T) {
|
|
// We need to ensure that the translate key partitions for each
|
|
// node are getting set as read-only based on the full cluster,
|
|
// not just the state of the cluster at the time of the individual
|
|
// node restart.
|
|
t.Run("RollingRestart", func(t *testing.T) {
|
|
t.Skip("skipping because disco needs asynchrounous restart")
|
|
// Start a 4-node cluster.
|
|
// Note that the prefix on the nodeID is intentional; it puts the
|
|
// nodes in a specific order which exercises the condition for
|
|
// which we are testing. In a normal use case, these would be
|
|
// randomly generated uuids, so this is mimicking that.
|
|
c := test.MustRunCluster(t, 4,
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(true),
|
|
pilosa.OptServerNodeID("2node0"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerNodeID("4node1"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerNodeID("3node2"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerNodeID("1node3"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
)
|
|
defer c.Close()
|
|
|
|
node0 := c.GetNode(0)
|
|
node1 := c.GetNode(1)
|
|
node2 := c.GetNode(2)
|
|
node3 := c.GetNode(3)
|
|
|
|
ctx := context.Background()
|
|
idx := "i"
|
|
|
|
// Create an index with keys.
|
|
if _, err := node0.API.CreateIndex(ctx, idx,
|
|
pilosa.IndexOptions{
|
|
Keys: true,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Stop the cluster.
|
|
if err := node0.Command.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := node1.Command.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := node2.Command.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := node3.Command.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Restart the nodes serially.
|
|
if err := node0.SoftOpen(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gossipSeeds := []string{node0.GossipAddress()}
|
|
|
|
node1.Config.Gossip.Seeds = gossipSeeds
|
|
if err := node1.SoftOpen(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
node2.Config.Gossip.Seeds = gossipSeeds
|
|
if err := node2.SoftOpen(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
node3.Config.Gossip.Seeds = gossipSeeds
|
|
if err := node3.SoftOpen(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Send a key translation request that triggers
|
|
// a read-only translate store error if the
|
|
// translate store sync is not reset correctly.
|
|
|
|
// Generate request body for translate row keys request
|
|
reqBody, err := node0.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
|
|
Index: idx,
|
|
Keys: []string{"a1"},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := node0.API.TranslateKeys(ctx, bytes.NewReader(reqBody)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestTranslation_KeyNotFound(t *testing.T) {
|
|
c := test.MustRunCluster(t, 4,
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(true),
|
|
pilosa.OptServerNodeID("node0"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerNodeID("node1"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerNodeID("node2"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerNodeID("node3"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
)
|
|
defer c.Close()
|
|
|
|
node0 := c.GetNode(0)
|
|
node1 := c.GetNode(1)
|
|
node2 := c.GetNode(2)
|
|
node3 := c.GetNode(3)
|
|
|
|
ctx := context.Background()
|
|
idx, fld := "i", "f"
|
|
// Create an index with keys.
|
|
if _, err := node0.API.CreateIndex(ctx, idx, pilosa.IndexOptions{Keys: true}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Create an index with keys.
|
|
if _, err := node0.API.CreateField(ctx, idx, fld, pilosa.OptFieldKeys()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// write a new key and get id
|
|
req, err := node0.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
|
|
Index: idx,
|
|
Field: fld,
|
|
Keys: []string{"k1"},
|
|
NotWritable: false,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if buf, err := node0.API.TranslateKeys(ctx, bytes.NewReader(req)); err != nil {
|
|
t.Fatal(err)
|
|
} else {
|
|
var resp pilosa.TranslateKeysResponse
|
|
if err = node0.API.Serializer.Unmarshal(buf, &resp); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id1 := resp.IDs[0]
|
|
|
|
// read non-existing key
|
|
req, err = node3.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
|
|
Index: idx,
|
|
Field: fld,
|
|
Keys: []string{"k2"},
|
|
NotWritable: true,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if buf, err = node3.API.TranslateKeys(ctx, bytes.NewReader(req)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = node3.API.Serializer.Unmarshal(buf, &resp); err != nil {
|
|
t.Fatal(err)
|
|
} else if resp.IDs != nil {
|
|
t.Fatalf("TranslateKeys(%+v): expected: nil, got: %d", string(req), resp)
|
|
}
|
|
|
|
req, err = node1.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
|
|
Index: idx,
|
|
Keys: []string{"k2"},
|
|
NotWritable: true,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if buf, err = node1.API.TranslateKeys(ctx, bytes.NewReader(req)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = node1.API.Serializer.Unmarshal(buf, &resp); err != nil {
|
|
t.Fatal(err)
|
|
} else if resp.IDs != nil {
|
|
t.Fatalf("TranslateKeys(%+v): expected: nil, got: %d", req, resp)
|
|
}
|
|
|
|
req, err = node2.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
|
|
Index: idx,
|
|
Field: fld,
|
|
Keys: []string{"k2", "k1"},
|
|
NotWritable: false,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if buf, err = node2.API.TranslateKeys(ctx, bytes.NewReader(req)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = node2.API.Serializer.Unmarshal(buf, &resp); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if resp.IDs[0] != id1+1 || resp.IDs[1] != id1 {
|
|
t.Fatalf("TranslateKeys(%+v): expected: %d,%d, got: %d,%d", req, id1+1, id1, resp.IDs[0], resp.IDs[1])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestInMemTranslateStore_ReadKey(t *testing.T) {
|
|
s := pilosa.NewInMemTranslateStore("IDX", "FLD", 0, topology.DefaultPartitionN)
|
|
|
|
id, err := s.TranslateKey("foo", false)
|
|
if err != pilosa.ErrTranslatingKeyNotFound {
|
|
t.Fatal(err)
|
|
}
|
|
if got, want := id, uint64(0); got != want {
|
|
t.Fatalf("TranslateKey()=%d, want %d", got, want)
|
|
}
|
|
|
|
// Ensure next key autoincrements.
|
|
if id, err = s.TranslateKey("foo", true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, want := id, uint64(1); got != want {
|
|
t.Fatalf("TranslateKey()=%d, want %d", got, want)
|
|
}
|
|
|
|
id1, err := s.TranslateKey("foo", false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, want := id1, id; got != want || id == 0 {
|
|
t.Fatalf("TranslateKey()=%d, want %d", got, want)
|
|
}
|
|
|
|
}
|
|
|
|
// Test index key translation replication under node failure.
|
|
func TestTranslation_Replication(t *testing.T) {
|
|
t.Run("Replication", func(t *testing.T) {
|
|
c := test.MustRunCluster(t, 3,
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(true),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
pilosa.OptServerReplicaN(2),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
pilosa.OptServerReplicaN(2),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
pilosa.OptServerReplicaN(2),
|
|
)},
|
|
)
|
|
defer c.Close()
|
|
|
|
node0 := c.GetNode(0)
|
|
node1 := c.GetNode(1)
|
|
|
|
ctx := context.Background()
|
|
idx := "i"
|
|
field := "f"
|
|
|
|
// Create an index with keys.
|
|
if _, err := node0.API.CreateIndex(ctx, idx,
|
|
pilosa.IndexOptions{
|
|
Keys: true,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if _, err := node0.API.CreateField(ctx, idx, field); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Write data on first node.
|
|
// these keys are a minimal example to reproduce the problem for the case of a 3-node cluster with replication factor 2
|
|
if _, err := node0.Queryf(t, idx, "", `
|
|
Set("x1", f=1)
|
|
Set("x2", f=1)
|
|
`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
exp := `{"results":[{"attrs":{},"columns":[],"keys":["x1","x2"]}]}`
|
|
|
|
if !test.CheckClusterState(node0, pilosa.ClusterStateNormal, 1000) {
|
|
t.Fatalf("unexpected node0 cluster state: %s", node0.API.State())
|
|
} else if !test.CheckClusterState(node1, pilosa.ClusterStateNormal, 1000) {
|
|
t.Fatalf("unexpected node1 cluster state: %s", node1.API.State())
|
|
}
|
|
|
|
// Verify the data exists
|
|
node0.QueryExpect(t, idx, "", `Row(f=1)`, exp)
|
|
|
|
// Kill one node.
|
|
if err := c.CloseAndRemove(1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify the data exists with one node down
|
|
node0.QueryExpect(t, idx, "", `Row(f=1)`, exp)
|
|
})
|
|
}
|
|
|
|
// Test key translation with multiple nodes.
|
|
func TestTranslation_Coordinator(t *testing.T) {
|
|
// Ensure that field key translations requests sent to
|
|
// non-coordinator nodes are forwarded to the coordinator.
|
|
t.Run("ForwardFieldKey", func(t *testing.T) {
|
|
t.Skip("Short term skip to avoid go 1.13 test Should remove ASAP")
|
|
// Start a 2-node cluster.
|
|
c := test.MustRunCluster(t, 2,
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(true),
|
|
pilosa.OptServerNodeID("node0"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerNodeID("node1"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
)
|
|
defer c.Close()
|
|
|
|
node0 := c.GetNode(0)
|
|
node1 := c.GetNode(1)
|
|
|
|
ctx := context.Background()
|
|
idx := "i"
|
|
fld := "f"
|
|
|
|
// Create an index without keys.
|
|
if _, err := node1.API.CreateIndex(ctx, idx,
|
|
pilosa.IndexOptions{
|
|
Keys: false,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Create a field with keys.
|
|
if _, err := node1.API.CreateField(ctx, idx, fld,
|
|
pilosa.OptFieldKeys(),
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
keys := []string{"one", "two", "three"}
|
|
for i := range keys {
|
|
pql := fmt.Sprintf(`Set(%d, %s="%s")`, i+1, fld, keys[i])
|
|
|
|
// Send a translation request to node1 (non-coordinator).
|
|
_, err := node1.API.Query(ctx,
|
|
&pilosa.QueryRequest{Index: idx, Query: pql},
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
for i := len(keys) - 1; i >= 0; i-- {
|
|
// Read the row and ensure the key was set.
|
|
qry := fmt.Sprintf(`Row(%s="%s")`, fld, keys[i])
|
|
resp, err := node0.API.Query(ctx,
|
|
&pilosa.QueryRequest{Index: idx, Query: qry},
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
row := resp.Results[0].(*pilosa.Row)
|
|
val := uint64(i + 1)
|
|
if cols := row.Columns(); !reflect.DeepEqual(cols, []uint64{val}) {
|
|
t.Fatalf("unexpected columns: %+v", cols)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestTranslation_TranslateIDsOnCluster(t *testing.T) {
|
|
c := test.MustRunCluster(t, 4,
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(true),
|
|
pilosa.OptServerNodeID("node0"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerNodeID("node1"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerNodeID("node2"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
[]server.CommandOption{
|
|
server.OptCommandServerOptions(
|
|
pilosa.OptServerIsCoordinator(false),
|
|
pilosa.OptServerNodeID("node3"),
|
|
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
|
|
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
|
|
)},
|
|
)
|
|
defer c.Close()
|
|
|
|
node0 := c.GetNode(0)
|
|
node3 := c.GetNode(3)
|
|
|
|
ctx := context.Background()
|
|
idx, fld := "i", "f"
|
|
// Create an index with keys.
|
|
if _, err := node0.API.CreateIndex(ctx, idx, pilosa.IndexOptions{Keys: true}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Create an index with keys.
|
|
if _, err := node0.API.CreateField(ctx, idx, fld, pilosa.OptFieldKeys()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
keys := []string{"k0", "k1", "k2", "k3", "k4", "k5", "k6", "k7", "k8", "k9"}
|
|
// write a new key and get id
|
|
req, err := node0.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
|
|
Index: idx,
|
|
Field: fld,
|
|
Keys: keys,
|
|
NotWritable: false,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if buf, err := node0.API.TranslateKeys(ctx, bytes.NewReader(req)); err != nil {
|
|
t.Fatal(err)
|
|
} else {
|
|
var (
|
|
respKeys pilosa.TranslateKeysResponse
|
|
respIDs pilosa.TranslateIDsResponse
|
|
)
|
|
if err = node0.API.Serializer.Unmarshal(buf, &respKeys); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ids := respKeys.IDs
|
|
|
|
// translate ids
|
|
req, err = node3.API.Serializer.Marshal(&pilosa.TranslateIDsRequest{
|
|
Index: idx,
|
|
Field: fld,
|
|
IDs: ids,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if buf, err = node3.API.TranslateIDs(ctx, bytes.NewReader(req)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = node3.API.Serializer.Unmarshal(buf, &respIDs); err != nil {
|
|
t.Fatal(err)
|
|
} else if !reflect.DeepEqual(respIDs.Keys, keys) {
|
|
t.Fatalf("TranslateIDs(%+v): expected: %+v, got: %+v", ids, keys, respIDs.Keys)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTranslation_Cluster_CreateFind(t *testing.T) {
|
|
c := test.MustRunCluster(t, 3)
|
|
defer c.Close()
|
|
|
|
c.CreateField(t, "i", pilosa.IndexOptions{Keys: true}, "f", pilosa.OptFieldKeys())
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
// Use the alphabet to test keys.
|
|
testKeys := make(map[string]struct{})
|
|
for i := 'a'; i <= 'z'; i++ {
|
|
testKeys[string(i)] = struct{}{}
|
|
}
|
|
|
|
t.Run("Index", func(t *testing.T) {
|
|
// Create all index keys, split across nodes.
|
|
{
|
|
parts := make([][]string, len(c.Nodes))
|
|
{
|
|
// Randomly partition the keys.
|
|
i := 0
|
|
for k := range testKeys {
|
|
parts[i%len(c.Nodes)] = append(parts[i%len(c.Nodes)], k)
|
|
i++
|
|
}
|
|
}
|
|
|
|
// Create some keys on each node.
|
|
var g errgroup.Group
|
|
defer g.Wait() //nolint:errcheck
|
|
for i, keys := range parts {
|
|
i, keys := i, keys
|
|
g.Go(func() error {
|
|
_, err := c.Nodes[i].API.CreateIndexKeys(ctx, "i", keys...)
|
|
return err
|
|
})
|
|
}
|
|
if err := g.Wait(); err != nil {
|
|
t.Errorf("creating keys: %v", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Check that all index keys exist, and consistently map to the same IDs.
|
|
{
|
|
// Convert the keys to a list.
|
|
keyList := make([]string, 0, len(testKeys))
|
|
for k := range testKeys {
|
|
keyList = append(keyList, k)
|
|
}
|
|
|
|
// Obtain authoritative translations for the keys.
|
|
translations, err := c.Nodes[0].API.FindIndexKeys(ctx, "i", keyList...)
|
|
if err != nil {
|
|
t.Errorf("obtaining authoritative translations: %v", err)
|
|
return
|
|
}
|
|
for _, k := range keyList {
|
|
if _, ok := translations[k]; !ok {
|
|
t.Errorf("key %q is missing", k)
|
|
}
|
|
}
|
|
|
|
// Check that all nodes agree on these translations.
|
|
var g errgroup.Group
|
|
defer g.Wait() //nolint:errcheck
|
|
for i, n := range c.Nodes {
|
|
i, api := i, n.API
|
|
g.Go(func() (err error) {
|
|
defer func() { err = errors.Wrapf(err, "translating on node %d", i) }()
|
|
localTranslations, err := api.FindIndexKeys(ctx, "i", keyList...)
|
|
if err != nil {
|
|
return errors.Wrap(err, "finding translations")
|
|
}
|
|
return compareTranslations(translations, localTranslations)
|
|
})
|
|
}
|
|
if err := g.Wait(); err != nil {
|
|
t.Errorf("finding keys: %v", err)
|
|
return
|
|
}
|
|
|
|
// Check that re-invoking create returns the original translations.
|
|
for i, n := range c.Nodes {
|
|
i, api := i, n.API
|
|
g.Go(func() (err error) {
|
|
defer func() { err = errors.Wrapf(err, "translating on node %d", i) }()
|
|
localTranslations, err := api.CreateIndexKeys(ctx, "i", keyList...)
|
|
if err != nil {
|
|
return errors.Wrap(err, "finding translations")
|
|
}
|
|
return compareTranslations(translations, localTranslations)
|
|
})
|
|
}
|
|
if err := g.Wait(); err != nil {
|
|
t.Errorf("checking re-create of keys: %v", err)
|
|
return
|
|
}
|
|
}
|
|
})
|
|
t.Run("Field", func(t *testing.T) {
|
|
// Create all field keys, split across nodes.
|
|
{
|
|
parts := make([][]string, len(c.Nodes))
|
|
{
|
|
// Randomly partition the keys.
|
|
i := 0
|
|
for k := range testKeys {
|
|
parts[i%len(c.Nodes)] = append(parts[i%len(c.Nodes)], k)
|
|
i++
|
|
}
|
|
}
|
|
|
|
// Create some keys on each node.
|
|
var g errgroup.Group
|
|
defer g.Wait() //nolint:errcheck
|
|
for i, keys := range parts {
|
|
i, keys := i, keys
|
|
g.Go(func() error {
|
|
_, err := c.Nodes[i].API.CreateFieldKeys(ctx, "i", "f", keys...)
|
|
return err
|
|
})
|
|
}
|
|
if err := g.Wait(); err != nil {
|
|
t.Errorf("creating keys: %v", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Check that all field keys exist, and consistently map to the same IDs.
|
|
{
|
|
// Convert the keys to a list.
|
|
keyList := make([]string, 0, len(testKeys))
|
|
for k := range testKeys {
|
|
keyList = append(keyList, k)
|
|
}
|
|
|
|
// Obtain authoritative translations for the keys.
|
|
translations, err := c.Nodes[0].API.FindFieldKeys(ctx, "i", "f", keyList...)
|
|
if err != nil {
|
|
t.Errorf("obtaining authoritative translations: %v", err)
|
|
return
|
|
}
|
|
for _, k := range keyList {
|
|
if _, ok := translations[k]; !ok {
|
|
t.Errorf("key %q is missing", k)
|
|
}
|
|
}
|
|
|
|
// Check that all nodes agree on these translations.
|
|
var g errgroup.Group
|
|
defer g.Wait() //nolint:errcheck
|
|
for i, n := range c.Nodes {
|
|
i, api := i, n.API
|
|
g.Go(func() (err error) {
|
|
defer func() { err = errors.Wrapf(err, "translating on node %d", i) }()
|
|
localTranslations, err := api.FindFieldKeys(ctx, "i", "f", keyList...)
|
|
if err != nil {
|
|
return errors.Wrap(err, "finding translations")
|
|
}
|
|
return compareTranslations(translations, localTranslations)
|
|
})
|
|
}
|
|
if err := g.Wait(); err != nil {
|
|
t.Errorf("finding keys: %v", err)
|
|
return
|
|
}
|
|
|
|
// Check that re-invoking create returns the original translations.
|
|
for i, n := range c.Nodes {
|
|
i, api := i, n.API
|
|
g.Go(func() (err error) {
|
|
defer func() { err = errors.Wrapf(err, "translating on node %d", i) }()
|
|
localTranslations, err := api.CreateFieldKeys(ctx, "i", "f", keyList...)
|
|
if err != nil {
|
|
return errors.Wrap(err, "finding translations")
|
|
}
|
|
return compareTranslations(translations, localTranslations)
|
|
})
|
|
}
|
|
if err := g.Wait(); err != nil {
|
|
t.Errorf("checking re-create of keys: %v", err)
|
|
return
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func compareTranslations(expected, got map[string]uint64) error {
|
|
for key, id := range got {
|
|
if realID, ok := expected[key]; !ok {
|
|
return errors.Errorf("unexpected key %q mapped to ID %d", key, id)
|
|
} else if id != realID {
|
|
return errors.Errorf("mismatched translation: expected %q:%d but got %q:%d", key, realID, key, id)
|
|
}
|
|
}
|
|
for key, realID := range expected {
|
|
if id, ok := got[key]; !ok {
|
|
return errors.Errorf("missing translation of key %q", key)
|
|
} else if id != realID {
|
|
return errors.Errorf("mismatched translation: expected %q:%d but got %q:%d", key, realID, key, id)
|
|
}
|
|
}
|
|
return nil
|
|
}
|