scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
featurebase/client/client_it_test.go
Seebs f6d17b1b58 refactor testing to share clusters more often 
When doing tests, we create a ton of one-off clusters. This
turns out to be expensive and slow. Fixing it is surprisingly hard.

Fundamentally: If we're sharing clusters, we need to use different
indexes for each test, to avoid clashes. This changes index names.
As a side-effect, this reorders many partition-based things, like
the order keys are returned in. Thus, to fix this, we change a lot
of tests to no longer depend on the *order* in which strings are
returned.

Having done that, we can also discard the ModHasher behavior, since
that only existed to allow us to reliably predict partitioning.

The basic design is as follows: Instead of a cluster being a
[]*Command, a "shareable" cluster is now a []*Command plus some
flags, and a "cluster" is a pointer to a possibly-shared cluster,
plus a link to the specific test using this specific cluster,
and correspondingly, its test name suitably coerced to be a valid
index name prefix.

The "test.Cluster" object now has methods to allow retrieving an
index name, and also implemnts fmt.Formatter to let you use,
e.g., `%i` with it in Sprintf to get "the index name, plus an i".
(This works for everything but %p and %T.)

This allows us to consistently rework all the many things that
use index names in a persistent way.

We also have `MustUnshared` and `MustRunUnsharedCluster` methods
which allow us to specify that a given test needs its own cluster
for some reason. For instance, the tests that want to run backups
need their own isolated cluster, and the tests that want to close
or reopen nodes need their own cluster because a reopened cluster
won't have working GRPC for some reason.

On "closing" a shared cluster (actually the test-specific wrapper
that reflects a given sharing), we delete any indexes starting with
that test's index name prefix. Otherwise, the huge pile of open
indexes prevents `go test -race` from working on MacOS, where we
run out of address space too quickly.

This is fairly enormous but most of the individual changes are
fairly trivial things like replacing the string "i" with "c.Idx()".

We also tweaked a test that failed for me a couple of times to
not depend on sort order.
2022-09-30 11:10:47 -07:00

893 lines
28 KiB
Go
Raw Blame History

// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package client
import (
"bytes"
"fmt"
"io"
"testing"
"time"
featurebase "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/disco"
pnet "github.com/featurebasedb/featurebase/v3/net"
"github.com/featurebasedb/featurebase/v3/roaring"
"github.com/featurebasedb/featurebase/v3/shardwidth"
"github.com/featurebasedb/featurebase/v3/test"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
var (
testIndex *Index
testIndexWithKeys *Index
testIndexWithKeysNoTrack *Index
testIndexAtomicRecord *Index
testIndexKeyTranslation *Index
testField *Field
testFieldTimestamp *Field
testFieldInt *Field
testFieldTimeQuantum *Field
testFieldInt0 *Field
testFieldInt1 *Field
)
func setup(t *testing.T, cli *Client) {
t.Helper()
testSchema := NewSchema()
testIndex = testSchema.Index("test-index")
testIndexWithKeys = testSchema.Index("test-index-keys", OptIndexKeys(true))
testIndexWithKeysNoTrack = testSchema.Index("test-index-keys-notrack",
OptIndexKeys(true),
OptIndexTrackExistence(false),
)
testField = testIndex.Field("test-field")
testFieldTimeQuantum = testIndex.Field("test-field-timequantum", OptFieldTypeTime(TimeQuantumYear))
testFieldTimestamp = testIndex.Field("test-field-timestamp", OptFieldTypeTimestamp(time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC), "s"))
testFieldInt = testIndex.Field("test-field-int", OptFieldTypeInt(0, 100000))
testIndexKeyTranslation = testSchema.Index("test-index-key-translation", OptIndexKeys(true))
testIndexAtomicRecord = testSchema.Index("test-index-atomic-record")
testFieldInt0 = testIndexAtomicRecord.Field("test-field-int0", OptFieldTypeInt(-1000, 1000))
testFieldInt1 = testIndexAtomicRecord.Field("test-field-int1", OptFieldTypeInt(-1000, 1000))
require.NoErrorf(t, cli.SyncSchema(testSchema), "SyncSchema")
}
func tearDown(t *testing.T, cli *Client) {
t.Helper()
for _, i := range []*Index{testIndex, testIndexWithKeys, testIndexWithKeysNoTrack, testIndexAtomicRecord, testIndexKeyTranslation} {
require.NoErrorf(t, cli.DeleteIndex(i), "DeleteIndex(%s)", i.name)
}
}
func TestClientAgainstCluster(t *testing.T) {
for size, replicaN := 3, 1; replicaN <= 2; replicaN++ {
testName := fmt.Sprintf("%d.%d", size, replicaN)
t.Run(testName, func(t *testing.T) {
// Start size.replicaN cluster
c := test.MustUnsharedCluster(t, size)
for _, n := range c.Nodes {
n.Config.Cluster.ReplicaN = replicaN
}
err := c.Start()
require.NoError(t, err, "Start cluster "+testName)
urls := make([]string, len(c.Nodes))
for i, n := range c.Nodes {
urls[i] = n.URL()
}
defer c.Close()
// Create a new client for the cluster
cli, err := newClientFromAddresses(urls, &ClientOptions{})
require.NoErrorf(t, err, "newClientFromAddresses(%v): %v", urls, err)
defer cli.Close()
t.Run("GetStatus", func(t *testing.T) {
status, err := cli.Status()
require.NoErrorf(t, err, "GET /status")
require.Equalf(t, disco.ClusterStateNormal, disco.ClusterState(status.State), "GET /status")
})
t.Run("QueryRow", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
resp, err := cli.Query(testField.Row(1))
require.NoErrorf(t, err, "Query Row")
require.NotNil(t, resp, "Response should not be nil")
})
t.Run("IntBase", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testIndex.Field("intbase", OptFieldTypeInt(-10, -5))
testIndex.Field("intbaseplus", OptFieldTypeInt(5, 10))
err = cli.SyncIndex(testIndex)
require.NoError(t, err)
schema, err := cli.Schema()
require.NoError(t, err)
if base := schema.Index("test-index").Field("intbase").Options().base; base != -5 {
t.Fatalf("unexpected base is not -5: %d", base)
}
if base := schema.Index("test-index").Field("intbaseplus").Options().base; base != 5 {
t.Fatalf("unexpected base is not 5: %d", base)
}
})
t.Run("QueryWithShards", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
shardWidth := uint64(1 << shardwidth.Exponent)
_, err := cli.Query(testField.Set(1, 1))
require.NoErrorf(t, err, "Set(1, %d)", 1)
_, err = cli.Query(testField.Set(1, shardWidth))
require.NoErrorf(t, err, "Set(1, %d)", shardWidth)
_, err = cli.Query(testField.Set(1, shardWidth*3))
require.NoErrorf(t, err, "Set(1, %d)", shardWidth*3)
resp, err := cli.Query(testField.Row(1), OptQueryShards(0, 3))
require.NoErrorf(t, err, "Row(1) OptQueryShards(0, 3)")
cols := resp.Result().Row().Columns
require.Equalf(t, []uint64{1, shardWidth * 3}, cols, "Unexpected results: %#v", cols)
})
t.Run("OrmCount", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldCount := testIndex.Field("test-field-count")
err := cli.EnsureField(testFieldCount)
require.NoError(t, err)
qry := testIndex.BatchQuery(
testFieldCount.Set(10, 20),
testFieldCount.Set(10, 21),
testFieldCount.Set(15, 25),
)
_, err = cli.Query(qry)
require.NoErrorf(t, err, "BatchQuery")
resp, err := cli.Query(testIndex.Count(testFieldCount.Row(10)))
require.NoErrorf(t, err, "Count")
require.Equalf(t, int64(2), resp.Result().Count(), "Count")
})
t.Run("DecimalField", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldDec := testIndex.Field("test-field-dec", OptFieldTypeDecimal(3))
err := cli.EnsureField(testFieldDec)
require.NoError(t, err)
sch, err := cli.Schema()
require.NoErrorf(t, err, "Schema")
idx := sch.indexes[testIndex.name]
opts := idx.Field(testFieldDec.name).Options()
require.Equalf(t, int64(3), opts.scale, "%s scale", testFieldDec.name)
})
t.Run("IntersectReturns", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldSegments := testIndex.Field("test-field-segments")
err := cli.EnsureField(testFieldSegments)
require.NoError(t, err)
qry1 := testIndex.BatchQuery(
testFieldSegments.Set(2, 10),
testFieldSegments.Set(2, 15),
testFieldSegments.Set(3, 10),
testFieldSegments.Set(3, 20),
)
_, err = cli.Query(qry1)
require.NoErrorf(t, err, "BatchQuery")
qry2 := testIndex.Intersect(testFieldSegments.Row(2), testFieldSegments.Row(3))
resp, err := cli.Query(qry2)
require.NoErrorf(t, err, "Intersect")
require.Equalf(t, 1, len(resp.Results()), "Intersect number of results")
require.Equalf(t, []uint64{10}, resp.Result().Row().Columns, "Intersect columns results")
})
t.Run("TopNReturns", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldTopN := testIndex.Field("test-field-topn")
err := cli.EnsureField(testFieldTopN)
require.NoError(t, err)
qry := testIndex.BatchQuery(
testFieldTopN.Set(10, 5),
testFieldTopN.Set(10, 10),
testFieldTopN.Set(10, 15),
testFieldTopN.Set(20, 5),
testFieldTopN.Set(30, 5),
)
_, err = cli.Query(qry)
require.NoErrorf(t, err, "BatchQuery")
// XXX: The following is required to make this test pass. See: https://github.com/featurebasedb/featurebase/issues/625
_, _, err = cli.HTTPRequest("POST", "/recalculate-caches", nil, nil)
require.NoErrorf(t, err, "POST /recalculate-caches")
resp, err := cli.Query(testFieldTopN.TopN(2))
require.NoErrorf(t, err, "TopN(2)")
items := resp.Result().CountItems()
require.Equalf(t, 2, len(items), "TopN result CountItems")
item := items[0]
require.Equalf(t, uint64(10), item.ID, "TopN result item[0].ID")
require.Equalf(t, uint64(3), item.Count, "TopN result item[0].Count")
})
t.Run("MinMaxRow", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldMinMax := testIndex.Field("test-field-minmax")
err := cli.EnsureField(testFieldMinMax)
require.NoError(t, err)
qry := testIndex.BatchQuery(
testFieldMinMax.Set(10, 5),
testFieldMinMax.Set(10, 10),
testFieldMinMax.Set(10, 15),
testFieldMinMax.Set(20, 5),
testFieldMinMax.Set(30, 5),
)
_, err = cli.Query(qry)
require.NoErrorf(t, err, "Setting bits")
resp, err := cli.Query(testFieldMinMax.MinRow())
require.NoErrorf(t, err, "MinRow")
min := resp.Result().CountItem().ID
require.Equalf(t, uint64(10), min, "Min")
resp, err = cli.Query(testFieldMinMax.MaxRow())
require.NoErrorf(t, err, "MaxRow")
max := resp.Result().CountItem().ID
require.Equalf(t, uint64(30), max, "Max")
})
t.Run("SetMutexField", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldMutex := testIndex.Field("test-field-mutex", OptFieldTypeMutex(CacheTypeDefault, 0))
err := cli.EnsureField(testFieldMutex)
require.NoError(t, err)
// can set mutex
_, err = cli.Query(testFieldMutex.Set(1, 100))
require.NoErrorf(t, err, "Set(1, 100)")
resp, err := cli.Query(testFieldMutex.Row(1))
require.NoErrorf(t, err, "Row(1)")
target := []uint64{100}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
// setting another row removes the previous
_, err = cli.Query(testFieldMutex.Set(42, 100))
require.NoErrorf(t, err, "Set(42, 100)")
resp, err = cli.Query(testIndex.BatchQuery(
testFieldMutex.Row(1),
testFieldMutex.Row(42),
))
require.NoErrorf(t, err, "BatchQuery")
target1 := []uint64(nil)
target42 := []uint64{100}
require.Equalf(t, target1, resp.Results()[0].Row().Columns, "Row Results[0] Columns")
require.Equalf(t, target42, resp.Results()[1].Row().Columns, "Row Results[1] Columns")
})
t.Run("SetBoolField", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldBool := testIndex.Field("test-field-bool", OptFieldTypeBool())
err := cli.EnsureField(testFieldBool)
require.NoError(t, err)
// can set bool
_, err = cli.Query(testFieldBool.Set(true, 100))
require.NoErrorf(t, err, "Set(true, 100)")
resp, err := cli.Query(testFieldBool.Row(true))
require.NoErrorf(t, err, "Row(true)")
target := []uint64{100}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
})
t.Run("ClearRowQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldClear := testIndex.Field("test-field-clear")
err := cli.EnsureField(testFieldClear)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldClear.Set(1, 100),
testFieldClear.Set(1, 200),
))
require.NoErrorf(t, err, "Set(1, 100) Set(1, 200)")
resp, err := cli.Query(testFieldClear.Row(1))
require.NoErrorf(t, err, "Row(1)")
target := []uint64{100, 200}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
_, err = cli.Query(testFieldClear.ClearRow(1))
require.NoErrorf(t, err, "ClearRow(1)")
resp, err = cli.Query(testFieldClear.Row(1))
require.NoErrorf(t, err, "Row(1)")
target = []uint64(nil)
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
})
t.Run("RowsQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldRows := testIndex.Field("test-field-rows")
err := cli.EnsureField(testFieldRows)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldRows.Set(1, 100),
testFieldRows.Set(1, 200),
testFieldRows.Set(2, 200),
))
require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)")
resp, err := cli.Query(testFieldRows.Rows())
require.NoErrorf(t, err, "Rows")
target := RowIdentifiersResult{
IDs: []uint64{1, 2},
}
require.Equalf(t, target, resp.Result().RowIdentifiers(), "RowIdentifiers Result")
})
t.Run("UnionRowsQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldRows := testIndex.Field("test-field-rows")
err := cli.EnsureField(testFieldRows)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldRows.Set(1, 100),
testFieldRows.Set(1, 200),
testFieldRows.Set(2, 200),
))
require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)")
resp, err := cli.Query(testFieldRows.Rows().Union())
require.NoErrorf(t, err, "Rows Union")
target := []uint64{100, 200}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
})
t.Run("LikeQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldLike := testIndex.Field("test-field-like", OptFieldKeys(true))
err := cli.EnsureField(testFieldLike)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldLike.Set("a", 100),
testFieldLike.Set("b", 200),
testFieldLike.Set("bc", 200),
))
require.NoErrorf(t, err, "Set(a, 100) Set(b, 200) Set(bc, 200)")
resp, err := cli.Query(testFieldLike.Like("b%"))
require.NoErrorf(t, err, `Like(b%)`)
target := RowIdentifiersResult{
Keys: []string{"b", "bc"},
}
require.Equalf(t, target, resp.Result().RowIdentifiers(), "RowIdentifiers Result")
})
t.Run("GroupByQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldGroupBy := testIndex.Field("test-field-group-by")
err := cli.EnsureField(testFieldGroupBy)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldGroupBy.Set(1, 100),
testFieldGroupBy.Set(1, 200),
testFieldGroupBy.Set(2, 200),
))
require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)")
resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows()))
require.NoErrorf(t, err, `Like(b%)`)
target := []GroupCount{
{Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 1}}, Count: 2},
{Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 2}}, Count: 1},
}
assertGroupBy(t, target, resp.Result().GroupCounts())
})
t.Run("GroupByQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldGroupBy := testIndex.Field("test-field-group-by-int", OptFieldTypeInt(-10, 10))
err := cli.EnsureField(testFieldGroupBy)
require.NoError(t, err)
_, err = cli.Query(testIndex.RawQuery(`
Set(0, test-field-group-by-int=1)
Set(1, test-field-group-by-int=2)
Set(2, test-field-group-by-int=-2)
Set(3, test-field-group-by-int=-1)
Set(4, test-field-group-by-int=4)
Set(10, test-field-group-by-int=0)
Set(100, test-field-group-by-int=0)
Set(1000, test-field-group-by-int=0)
Set(10000, test-field-group-by-int=0)
Set(100000, test-field-group-by-int=0)
`))
require.NoError(t, err, "Set(0..100000)")
resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows()))
require.NoErrorf(t, err, `GroupBy(Rows)`)
var a, b, c, d, e, f int64 = -2, -1, 0, 1, 2, 4
target := []GroupCount{
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &a}}, Count: 1},
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &b}}, Count: 1},
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &c}}, Count: 5},
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &d}}, Count: 1},
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &e}}, Count: 1},
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &f}}, Count: 1},
}
assertGroupBy(t, target, resp.Result().GroupCounts())
})
t.Run("CreateDeleteIndexField", func(t *testing.T) {
tmpIndex := NewIndex("tmp-index")
tmpField := tmpIndex.Field("tmp-field")
err := cli.CreateIndex(tmpIndex)
require.NoError(t, err)
err = cli.CreateField(tmpField)
require.NoError(t, err)
err = cli.DeleteField(tmpField)
require.NoError(t, err)
err = cli.DeleteIndex(tmpIndex)
require.NoError(t, err)
})
t.Run("ErrorCreatingIndexField", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
require.ErrorIs(t, cli.CreateIndex(testIndex), ErrIndexExists)
require.ErrorIs(t, cli.CreateField(testField), ErrFieldExists)
})
t.Run("Failover", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
uri, _ := pnet.NewURIFromAddress("does-not-resolve.foo.bar")
tmpcli, _ := NewClient(NewClusterWithHost(uri, uri, uri, uri), OptClientRetries(0))
_, err := tmpcli.Query(testIndex.All())
require.Error(t, err, ErrHTTPRequest)
})
t.Run("InvalidQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
_, _, err := cli.HTTPRequest("INVALID METHOD", "/foo", nil, nil)
require.Error(t, err)
_, err = cli.Query(testIndex.RawQuery("Invalid query"))
require.Error(t, err)
})
t.Run("Sync", func(t *testing.T) {
testIndexRemote := NewIndex("test-index-remote")
err := cli.EnsureIndex(testIndexRemote)
require.NoError(t, err)
testFieldRemote := testIndexRemote.Field("test-field-remote")
err = cli.EnsureField(testFieldRemote)
require.NoError(t, err)
schema := NewSchema()
idx1 := schema.Index("index-1")
idx1.Field("field-1-1")
idx1.Field("field-1-2")
idx2 := schema.Index("index-2")
idx2.Field("field-2-1")
schema.Index(testIndexRemote.Name())
err = cli.SyncSchema(schema)
require.NoError(t, err)
err = cli.DeleteIndex(testIndexRemote)
require.NoError(t, err)
err = cli.DeleteIndex(idx1)
require.NoError(t, err)
err = cli.DeleteIndex(idx2)
require.NoError(t, err)
})
t.Run("FetchFragmentNodes", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
nodes, err := cli.fetchFragmentNodes(testIndex.Name(), 0)
require.NoErrorf(t, err, "fetchFragmentNodes(%s, 0)", testIndex.name)
require.Equalf(t, replicaN, len(nodes), "len(nodes)")
// running the same for coverage
nodes, err = cli.fetchFragmentNodes(testIndex.Name(), 0)
require.NoErrorf(t, err, "fetchFragmentNodes(%s, 0)", testIndex.name)
require.Equalf(t, replicaN, len(nodes), "len(nodes)")
})
t.Run("RowRangeQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldRange := testIndex.Field("test-field-range", OptFieldTypeTime(TimeQuantumMonthDayHour))
err := cli.EnsureField(testFieldRange)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldRange.SetTimestamp(10, 100, time.Date(2017, time.January, 1, 0, 0, 0, 0, time.UTC)),
testFieldRange.SetTimestamp(10, 100, time.Date(2018, time.January, 1, 0, 0, 0, 0, time.UTC)),
testFieldRange.SetTimestamp(10, 100, time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC)),
))
require.NoErrorf(t, err, "BatchQuery SetTimestamp")
start := time.Date(2017, time.January, 5, 0, 0, 0, 0, time.UTC)
end := time.Date(2018, time.January, 5, 0, 0, 0, 0, time.UTC)
resp, err := cli.Query(testFieldRange.RowRange(10, start, end))
require.NoErrorf(t, err, "RowRange(10, %v, %v)", start, end)
target := []uint64{100}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
})
t.Run("StoreQuery", func(t *testing.T) {
schema := NewSchema()
testIndexStore := schema.Index("test-index-store")
testFieldFrom := testIndexStore.Field("test-field-from")
testFieldTo := testIndexStore.Field("test-field-to")
err := cli.SyncSchema(schema)
require.NoError(t, err)
defer func() {
cerr := cli.DeleteIndex(testIndexStore)
require.NoErrorf(t, cerr, "failed to delete index: %v", testIndexStore.name)
}()
_, err = cli.Query(testIndexStore.BatchQuery(
testFieldFrom.Set(10, 100),
testFieldFrom.Set(10, 200),
testFieldTo.Store(testFieldFrom.Row(10), 1),
))
require.NoErrorf(t, err, "Set(10, 100) Set(10, 200) Store(Row(10), 1)")
resp, err := cli.Query(testFieldTo.Row(1))
require.NoErrorf(t, err, "Row(1)")
target := []uint64{100, 200}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
})
t.Run("MultipleClientKeyQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldMultiClient := testIndexWithKeys.Field("test-field-multiclient")
err := cli.EnsureField(testFieldMultiClient)
require.NoError(t, err)
eg := &errgroup.Group{}
for i := 0; i < 10; i++ {
rowID := uint64(i)
eg.Go(func() error {
_, e := cli.Query(testFieldMultiClient.Set(rowID, "col"))
return e
})
}
require.NoError(t, eg.Wait())
})
t.Run("ExportRowIDColumnID", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldExport := testIndex.Field("test-field-export")
err := cli.EnsureField(testFieldExport)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldExport.Set(1, 1),
testFieldExport.Set(1, 10),
testFieldExport.Set(2, 1048577),
), nil)
require.NoErrorf(t, err, "Set(1, 1) Set(1, 10) Set(2, 1048577)")
r, err := cli.ExportField(testFieldExport)
require.NoErrorf(t, err, "ExportField")
b, err := io.ReadAll(r)
require.NoError(t, err)
target := "1,1\n1,10\n2,1048577\n"
require.Equalf(t, target, string(b), "Export Field Response")
})
t.Run("ExportRowIDColumnKey", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldExport := testIndexWithKeys.Field("test-field-export")
err := cli.EnsureField(testFieldExport)
require.NoError(t, err)
_, err = cli.Query(testIndexWithKeys.BatchQuery(
testFieldExport.Set(1, "one"),
testFieldExport.Set(1, "ten"),
testFieldExport.Set(2, "big-number"),
), nil)
require.NoErrorf(t, err, "Set(1, one) Set(1, ten) Set(2, big-number)")
r, err := cli.ExportField(testFieldExport)
require.NoErrorf(t, err, "ExportField")
b, err := io.ReadAll(r)
require.NoError(t, err)
target := "1,one\n1,ten\n2,big-number\n"
require.Equalf(t, target, string(b), "Export Field Response")
})
t.Run("TranslateRowKeys", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldTranslate := testIndexKeyTranslation.Field("test-field-translate", OptFieldKeys(true))
err := cli.EnsureField(testFieldTranslate)
require.NoError(t, err)
trans, err := cli.CreateFieldKeys(testFieldTranslate, "key1", "key2")
require.NoErrorf(t, err, "CreateFieldKeys")
target := map[string]uint64{"key1": 1, "key2": 2}
require.Equalf(t, target, trans, "CreateFieldKeys")
trans, err = cli.FindFieldKeys(testFieldTranslate, "key1", "key2", "key3")
require.NoErrorf(t, err, "FindFieldKeys")
require.Equalf(t, target, trans, "FindFieldKeys")
})
t.Run("TranslateColKeys", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
created, err := cli.CreateIndexKeys(testIndexKeyTranslation, "key1", "key2")
require.NoErrorf(t, err, "CreateIndexKeys")
if _, ok := created["key1"]; !ok {
t.Error("key1 missing")
}
if _, ok := created["key2"]; !ok {
t.Error("key2 missing")
}
found, err := cli.FindIndexKeys(testIndexKeyTranslation, "key1", "key2", "key3")
require.NoErrorf(t, err, "FindIndexKeys")
require.Equalf(t, created, found, "IndexKeys")
})
t.Run("Transactions", func(t *testing.T) {
trns, err := cli.StartTransaction("blah", time.Minute, false, time.Minute)
require.NoErrorf(t, err, "StartTransaction(blah)")
require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID")
require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
require.Truef(t, trns.Active, "TranslateColumnKeys Active")
trnsMap, err := cli.Transactions()
require.NoErrorf(t, err, "Transactions")
require.Equalf(t, 1, len(trnsMap), "Transactions len")
require.Truef(t, trnsMap["blah"].Active, "Transactions Active")
trns, err = cli.GetTransaction("blah")
require.NoErrorf(t, err, "GetTransaction(blah)")
require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID")
require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
require.Truef(t, trns.Active, "TranslateColumnKeys Active")
trns, err = cli.FinishTransaction("blah")
require.NoErrorf(t, err, "FinishTransaction(blah)")
require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID")
require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
require.Truef(t, trns.Active, "TranslateColumnKeys Active")
})
t.Run("ImportRoaringShard", func(t *testing.T) {
setup(t, cli)
shardWidth := uint64(1 << shardwidth.Exponent)
bitmap := roaring.NewBitmap(1, shardWidth*2+1, shardWidth*3+1)
buf := &bytes.Buffer{}
_, err := bitmap.WriteTo(buf)
if err != nil {
t.Fatalf("serializing bitmap: %v", err)
}
request := &featurebase.ImportRoaringShardRequest{
Remote: true,
Views: []featurebase.RoaringUpdate{
{
Field: "test-field",
View: "standard",
Set: buf.Bytes(),
},
{
Field: "test-field-timestamp",
View: "bsig_test-field-timestamp",
Set: buf.Bytes(),
},
{
Field: "test-field-int",
View: "bsig_test-field-int",
Set: buf.Bytes(),
},
},
}
err = cli.ImportRoaringShard("test-index", 3, request)
if err != nil {
t.Fatalf("import-roaring-shard: %v", err)
}
if resp, err := cli.Query(testField.Row(2)); err != nil {
t.Fatalf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 {
t.Fatalf("unexpected result: %v", res)
}
if resp, err := cli.Query(testFieldInt.NotNull()); err != nil {
t.Fatalf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 {
t.Fatalf("unexpected result: %v", res)
}
if resp, err := cli.Query(testFieldTimestamp.NotNull()); err != nil {
t.Fatalf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 {
t.Fatalf("unexpected result: %v", res)
}
if resp, err := cli.Query(testIndex.RawQuery("Row(test-field-timestamp>'1969-12-31T23:59:59Z')")); err != nil {
t.Fatalf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 {
t.Fatalf("unexpected result: %v", res)
}
// now write more data
bitmap = roaring.NewBitmap(1, 2, shardWidth*3+1, shardWidth*3+2)
buf = &bytes.Buffer{}
_, err = bitmap.WriteTo(buf)
if err != nil {
t.Fatalf("serializing bitmap: %v", err)
}
request = &featurebase.ImportRoaringShardRequest{
Remote: true,
Views: []featurebase.RoaringUpdate{
{
Field: "test-field",
View: "standard",
Set: buf.Bytes(),
},
{
Field: "test-field-timestamp",
View: "bsig_test-field-timestamp",
Set: buf.Bytes(),
},
{
Field: "test-field-int",
View: "bsig_test-field-int",
Set: buf.Bytes(),
},
},
}
err = cli.ImportRoaringShard("test-index", 3, request)
if err != nil {
t.Fatalf("import-roaring-shard: %v", err)
}
if resp, err := cli.Query(testField.Row(3)); err != nil {
t.Errorf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 {
t.Errorf("unexpected result: %v", res)
}
if resp, err := cli.Query(testFieldInt.NotNull()); err != nil {
t.Errorf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 {
t.Errorf("unexpected result: %v", res)
}
if resp, err := cli.Query(testFieldTimestamp.NotNull()); err != nil {
t.Errorf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 {
t.Errorf("unexpected result: %v", res)
}
if resp, err := cli.Query(testIndex.RawQuery("Row(test-field-timestamp>'1969-12-31T23:59:59Z')")); err != nil {
t.Errorf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 {
t.Errorf("unexpected result: %v", res)
}
})
})
}
}
func assertGroupBy(t *testing.T, expected, results []GroupCount) {
t.Helper()
require.Equalf(t, len(expected), len(results), "number of groupings mismatch")
for i, result := range results {
require.Equalf(t, expected[i], result, "unexpected result at %d", i)
}
}
Reference in a new issue View git blame Copy permalink