mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
772496b440
This adds a shard-based import endpoint which takes bitmap data for all field types and imports data for the whole shard transactionally. It uses the BitmapRewriter interface to try to intelligently allow for setting and clearing bits simultaneously without multiple writes which is especially helpful when ingesting into int-like fields, but also allows clear-and-then-set behavior for set fields.
1389 lines
37 KiB
Go
1389 lines
37 KiB
Go
// Copyright 2021 Molecula Corp. All rights reserved.
|
|
|
|
package client
|
|
|
|
import (
|
|
"math/rand"
|
|
"reflect"
|
|
"sort"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/molecula/featurebase/v3/test"
|
|
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
func NewTestClient(t *testing.T, c *test.Cluster) *Client {
|
|
client, err := NewClient(c.Nodes[0].URL())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return client
|
|
}
|
|
|
|
func TestAgainstCluster(t *testing.T) {
|
|
c := test.MustRunCluster(t, 1)
|
|
defer c.Close()
|
|
client := NewTestClient(t, c)
|
|
|
|
t.Run("string-slice-combos", func(t *testing.T) { testStringSliceCombos(t, c, client) })
|
|
t.Run("import-batch-ints", func(t *testing.T) { testImportBatchInts(t, c, client) })
|
|
t.Run("import-batch-sorting", func(t *testing.T) { testImportBatchSorting(t, c, client) })
|
|
t.Run("test-trim-null", func(t *testing.T) { testTrimNull(t, c, client) })
|
|
t.Run("test-string-slice-empty-and-nil", func(t *testing.T) { testStringSliceEmptyAndNil(t, c, client) })
|
|
t.Run("test-string-slice", func(t *testing.T) { testStringSlice(t, c, client) })
|
|
t.Run("test-single-clear-batch-regression", func(t *testing.T) { testSingleClearBatchRegression(t, c, client) })
|
|
t.Run("test-batches", func(t *testing.T) { testBatches(t, c, client) })
|
|
t.Run("batches-strings-ids", func(t *testing.T) { testBatchesStringIDs(t, c, client) })
|
|
t.Run("test-batch-staleness", func(t *testing.T) { testBatchStaleness(t, c, client) })
|
|
t.Run("test-import-batch-multiple-ints", func(t *testing.T) { testImportBatchMultipleInts(t, c, client) })
|
|
}
|
|
|
|
func testStringSliceCombos(t *testing.T, c *test.Cluster, client *Client) {
|
|
schema := NewSchema()
|
|
idx := schema.Index("test-string-slice-combos")
|
|
fields := make([]*Field, 1)
|
|
fields[0] = idx.Field("a1", OptFieldKeys(true), OptFieldTypeSet(CacheTypeRanked, 100))
|
|
err := client.SyncSchema(schema)
|
|
if err != nil {
|
|
t.Fatalf("syncing schema: %v", err)
|
|
}
|
|
defer func() {
|
|
err := client.DeleteIndex(idx)
|
|
if err != nil {
|
|
t.Logf("problem cleaning up from test: %v", err)
|
|
}
|
|
}()
|
|
|
|
b, err := NewBatch(client, 5, idx, fields)
|
|
if err != nil {
|
|
t.Fatalf("creating new batch: %v", err)
|
|
}
|
|
|
|
records := []Row{
|
|
{ID: uint64(0), Values: []interface{}{[]string{"a", "b", "c"}}},
|
|
{ID: uint64(1), Values: []interface{}{[]string{"z"}}},
|
|
{ID: uint64(2), Values: []interface{}{[]string{}}},
|
|
{ID: uint64(3), Values: []interface{}{[]string{"q", "r", "s", "t", "c"}}},
|
|
{ID: uint64(4), Values: []interface{}{nil}},
|
|
{ID: uint64(5), Values: []interface{}{[]string{"a", "b", "c"}}},
|
|
{ID: uint64(6), Values: []interface{}{[]string{"a", "b", "c"}}},
|
|
{ID: uint64(7), Values: []interface{}{[]string{"z"}}},
|
|
{ID: uint64(8), Values: []interface{}{[]string{}}},
|
|
{ID: uint64(9), Values: []interface{}{[]string{"q", "r", "s", "t"}}},
|
|
{ID: uint64(10), Values: []interface{}{nil}},
|
|
{ID: uint64(11), Values: []interface{}{[]string{"a", "b", "c"}}},
|
|
{ID: uint64(12), Values: []interface{}{[]string{}}},
|
|
{ID: uint64(13), Values: []interface{}{[]string{}}},
|
|
}
|
|
|
|
err = ingestRecords(records, b)
|
|
if err != nil {
|
|
t.Fatalf("importing: %v", err)
|
|
}
|
|
|
|
a1 := fields[0]
|
|
|
|
result := tq(t, client, a1.TopN(10))
|
|
rez := sortableCRI(result.CountItems())
|
|
sort.Sort(rez)
|
|
exp := sortableCRI{
|
|
{Key: "a", Count: 4},
|
|
{Key: "b", Count: 4},
|
|
{Key: "c", Count: 5},
|
|
{Key: "q", Count: 2},
|
|
{Key: "r", Count: 2},
|
|
{Key: "s", Count: 2},
|
|
{Key: "t", Count: 2},
|
|
{Key: "z", Count: 2},
|
|
}
|
|
sort.Sort(exp)
|
|
errorIfNotEqual(t, exp, rez)
|
|
|
|
result = tq(t, client, a1.Row("a"))
|
|
errorIfNotEqual(t, result.Row().Columns, []uint64{0, 5, 6, 11})
|
|
result = tq(t, client, a1.Row("b"))
|
|
errorIfNotEqual(t, result.Row().Columns, []uint64{0, 5, 6, 11})
|
|
result = tq(t, client, a1.Row("c"))
|
|
errorIfNotEqual(t, result.Row().Columns, []uint64{0, 3, 5, 6, 11})
|
|
result = tq(t, client, a1.Row("z"))
|
|
errorIfNotEqual(t, result.Row().Columns, []uint64{1, 7})
|
|
result = tq(t, client, a1.Row("q"))
|
|
errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9})
|
|
result = tq(t, client, a1.Row("r"))
|
|
errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9})
|
|
result = tq(t, client, a1.Row("s"))
|
|
errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9})
|
|
result = tq(t, client, a1.Row("t"))
|
|
errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9})
|
|
|
|
result = tq(t, client, idx.RawQuery("Count(All())"))
|
|
errorIfNotEqual(t, result.Count(), int64(14))
|
|
}
|
|
|
|
func errorIfNotEqual(t *testing.T, exp, got interface{}) {
|
|
t.Helper()
|
|
if !reflect.DeepEqual(exp, got) {
|
|
t.Errorf("unequal exp/got:\n%v\n%v", exp, got)
|
|
}
|
|
}
|
|
|
|
type sortableCRI []CountResultItem
|
|
|
|
func (s sortableCRI) Len() int { return len(s) }
|
|
func (s sortableCRI) Less(i, j int) bool {
|
|
if s[i].Count != s[j].Count {
|
|
return s[i].Count > s[j].Count
|
|
}
|
|
if s[i].ID != s[j].ID {
|
|
return s[i].ID < s[j].ID
|
|
}
|
|
if s[i].Key != s[j].Key {
|
|
return s[i].Key < s[j].Key
|
|
}
|
|
return true
|
|
}
|
|
func (s sortableCRI) Swap(i, j int) {
|
|
s[i], s[j] = s[j], s[i]
|
|
}
|
|
|
|
func tq(t *testing.T, client *Client, query PQLQuery) QueryResult {
|
|
resp, err := client.Query(query)
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
return resp.Results()[0]
|
|
}
|
|
|
|
func ingestRecords(records []Row, batch *Batch) error {
|
|
for _, rec := range records {
|
|
err := batch.Add(rec)
|
|
if err == ErrBatchNowFull {
|
|
err = batch.Import()
|
|
if err != nil {
|
|
return errors.Wrap(err, "importing batch")
|
|
}
|
|
} else if err != nil {
|
|
return errors.Wrap(err, "while adding record")
|
|
}
|
|
}
|
|
if batch.Len() > 0 {
|
|
err := batch.Import()
|
|
if err != nil {
|
|
return errors.Wrap(err, "importing batch")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func testImportBatchInts(t *testing.T, c *test.Cluster, client *Client) {
|
|
schema := NewSchema()
|
|
idx := schema.Index("test-import-batch-ints")
|
|
field := idx.Field("anint", OptFieldTypeInt())
|
|
err := client.SyncSchema(schema)
|
|
if err != nil {
|
|
t.Fatalf("syncing schema: %v", err)
|
|
}
|
|
|
|
b, err := NewBatch(client, 3, idx, []*Field{field})
|
|
if err != nil {
|
|
t.Fatalf("getting batch: %v", err)
|
|
}
|
|
|
|
r := Row{Values: make([]interface{}, 1)}
|
|
|
|
for i := uint64(0); i < 3; i++ {
|
|
r.ID = i
|
|
r.Values[0] = int64(i)
|
|
err := b.Add(r)
|
|
if err != nil && err != ErrBatchNowFull {
|
|
t.Fatalf("adding to batch: %v", err)
|
|
}
|
|
}
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("importing: %v", err)
|
|
}
|
|
|
|
r.ID = uint64(0)
|
|
r.Values[0] = nil
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("adding after import: %v", err)
|
|
}
|
|
r.ID = uint64(1)
|
|
r.Values[0] = int64(7)
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("adding second after import: %v", err)
|
|
}
|
|
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("second import: %v", err)
|
|
}
|
|
|
|
resp, err := client.Query(idx.BatchQuery(field.Equals(0), field.Equals(7), field.Equals(2)))
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
|
|
for i, result := range resp.Results() {
|
|
if !reflect.DeepEqual(result.Row().Columns, []uint64{uint64(i)}) {
|
|
t.Errorf("expected %v for %d, but got %v", []uint64{uint64(i)}, i, result.Row().Columns)
|
|
}
|
|
}
|
|
}
|
|
|
|
func testImportBatchSorting(t *testing.T, c *test.Cluster, client *Client) {
|
|
schema := NewSchema()
|
|
idx := schema.Index("test-import-batch-sorting")
|
|
field := idx.Field("anint", OptFieldTypeInt())
|
|
field2 := idx.Field("amutex", OptFieldTypeMutex(CacheTypeNone, 0))
|
|
err := client.SyncSchema(schema)
|
|
if err != nil {
|
|
t.Fatalf("syncing schema: %v", err)
|
|
}
|
|
|
|
b, err := NewBatch(client, 100, idx, []*Field{field, field2})
|
|
if err != nil {
|
|
t.Fatalf("getting batch: %v", err)
|
|
}
|
|
|
|
r := Row{Values: make([]interface{}, 2)}
|
|
|
|
rnd := rand.New(rand.NewSource(7))
|
|
|
|
// generate 100 records randomly spread/ordered across multiple
|
|
// shards to test sorting on int/mutex fields
|
|
for i := 0; i < 100; i++ {
|
|
id := rnd.Intn(10_000_000)
|
|
r.ID = uint64(id)
|
|
r.Values[0] = int64(id)
|
|
r.Values[1] = uint64(id)
|
|
err := b.Add(r)
|
|
if err != nil && err != ErrBatchNowFull {
|
|
t.Fatalf("adding to batch: %v", err)
|
|
}
|
|
}
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("importing: %v", err)
|
|
}
|
|
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("second import: %v", err)
|
|
}
|
|
|
|
resp, err := client.Query(idx.RawQuery("Count(All())"))
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
if res := resp.Results()[0]; res.Count() != 100 {
|
|
t.Fatalf("unexpected result: %+v", res)
|
|
}
|
|
}
|
|
|
|
func testTrimNull(t *testing.T, c *test.Cluster, client *Client) {
|
|
schema := NewSchema()
|
|
idx := schema.Index("test-trim-null")
|
|
field := idx.Field("empty", OptFieldTypeInt())
|
|
err := client.SyncSchema(schema)
|
|
if err != nil {
|
|
t.Fatalf("syncing schema: %v", err)
|
|
}
|
|
defer func() {
|
|
err := client.DeleteIndex(idx)
|
|
if err != nil {
|
|
t.Logf("problem cleaning up from test: %v", err)
|
|
}
|
|
}()
|
|
b, err := NewBatch(client, 3, idx, []*Field{field})
|
|
if err != nil {
|
|
t.Fatalf("getting batch: %v", err)
|
|
}
|
|
b.nullIndices = make(map[string][]uint64, 1)
|
|
b.nullIndices[field.Name()] = []uint64{0, 1, 2}
|
|
r := Row{Values: make([]interface{}, 1)}
|
|
for i := 0; i < 3; i++ {
|
|
r.ID = uint64(i)
|
|
r.Values[0] = int64(i)
|
|
err := b.Add(r)
|
|
if err != nil && err != ErrBatchNowFull {
|
|
t.Fatalf("adding to batch: %v", err)
|
|
}
|
|
}
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("importing: %v", err)
|
|
}
|
|
resp, err := client.Query(idx.BatchQuery(field.Equals(0), field.Equals(1), field.Equals(2)))
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
for i, result := range resp.Results() {
|
|
if !reflect.DeepEqual(result.Row().Columns, []uint64(nil)) {
|
|
t.Errorf("expected %#v for %d, but got %#v", []uint64(nil), i, result.Row().Columns)
|
|
}
|
|
}
|
|
|
|
b, err = NewBatch(client, 4, idx, []*Field{field})
|
|
if err != nil {
|
|
t.Fatalf("getting batch: %v", err)
|
|
}
|
|
r = Row{Values: make([]interface{}, 1)}
|
|
for i := 10; i < 40; i += 10 {
|
|
r.ID = uint64(i)
|
|
r.Values[0] = int64(i)
|
|
err := b.Add(r)
|
|
if err != nil && err != ErrBatchNowFull {
|
|
t.Fatalf("adding to batch: %v", err)
|
|
}
|
|
}
|
|
|
|
r.ID = uint64(40)
|
|
r.Values[0] = nil
|
|
err = b.Add(r)
|
|
if err != nil && err != ErrBatchNowFull {
|
|
t.Fatalf("adding to batch: %v", err)
|
|
}
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("importing: %v", err)
|
|
}
|
|
|
|
resp, err = client.Query(idx.BatchQuery(field.Equals(10), field.Equals(40), field.Equals(20), field.Equals(30)))
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
for i, result := range resp.Results() {
|
|
if i == 1 {
|
|
if !reflect.DeepEqual(result.Row().Columns, []uint64(nil)) {
|
|
t.Errorf("expected %#v for %d, but got %#v", []uint64(nil), i, result.Row().Columns)
|
|
}
|
|
} else {
|
|
if !reflect.DeepEqual(result.Row().Columns, []uint64{result.Row().Columns[0]}) {
|
|
t.Errorf("expected %#v for %d, but got %#v", []uint64{result.Row().Columns[0]}, i, result.Row().Columns)
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func testStringSliceEmptyAndNil(t *testing.T, c *test.Cluster, client *Client) {
|
|
schema := NewSchema()
|
|
idx := schema.Index("test-string-slice-nil")
|
|
fields := make([]*Field, 1)
|
|
fields[0] = idx.Field("strslice", OptFieldKeys(true), OptFieldTypeSet(CacheTypeRanked, 100))
|
|
err := client.SyncSchema(schema)
|
|
if err != nil {
|
|
t.Fatalf("syncing schema: %v", err)
|
|
}
|
|
defer func() {
|
|
err := client.DeleteIndex(idx)
|
|
if err != nil {
|
|
t.Logf("problem cleaning up from test: %v", err)
|
|
}
|
|
}()
|
|
|
|
// first create a batch and test adding a single value with empty
|
|
// string - this failed with a translation error at one point, and
|
|
// how we catch it and treat it like a nil.
|
|
b, err := NewBatch(client, 2, idx, fields)
|
|
if err != nil {
|
|
t.Fatalf("creating new batch: %v", err)
|
|
}
|
|
r := Row{Values: make([]interface{}, len(fields))}
|
|
r.ID = uint64(1)
|
|
r.Values[0] = ""
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("adding: %v", err)
|
|
}
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("importing: %v", err)
|
|
}
|
|
|
|
// now create a batch and add a mixture of string slice values
|
|
b, err = NewBatch(client, 6, idx, fields)
|
|
if err != nil {
|
|
t.Fatalf("creating new batch: %v", err)
|
|
}
|
|
r = Row{Values: make([]interface{}, len(fields))}
|
|
r.ID = uint64(0)
|
|
r.Values[0] = []string{"a"}
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("adding to batch: %v", err)
|
|
}
|
|
|
|
r.ID = uint64(1)
|
|
r.Values[0] = nil
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("adding batch with nil stringslice to r: %v", err)
|
|
}
|
|
|
|
r.ID = uint64(2)
|
|
r.Values[0] = []string{"a", "b", "z"}
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("adding batch with idslice to r: %v", err)
|
|
}
|
|
|
|
r.ID = uint64(3)
|
|
r.Values[0] = []string{"b", "c"}
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("adding batch with stringslice to r: %v", err)
|
|
}
|
|
|
|
r.ID = uint64(4)
|
|
r.Values[0] = []string{}
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("adding batch with stringslice to r: %v", err)
|
|
}
|
|
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("importing: %v", err)
|
|
}
|
|
|
|
rows := []interface{}{"a", "b", "c", "z"}
|
|
resp, err := client.Query(idx.BatchQuery(fields[0].Row(rows[0]), fields[0].Row(rows[1]), fields[0].Row(rows[2]), fields[0].Row(rows[3])))
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
|
|
// TODO test is flaky because we can't guarantee what a,b,c map to
|
|
expectations := [][]uint64{{0, 2}, {2, 3}, {3}, {2}}
|
|
for i, re := range resp.Results() {
|
|
if !reflect.DeepEqual(re.Row().Columns, expectations[i]) {
|
|
t.Errorf("expected row %v to have columns %v, but got %v", rows[i], expectations[i], re.Row().Columns)
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func testStringSlice(t *testing.T, c *test.Cluster, client *Client) {
|
|
schema := NewSchema()
|
|
idx := schema.Index("test-string-slice")
|
|
fields := make([]*Field, 1)
|
|
fields[0] = idx.Field("strslice", OptFieldKeys(true), OptFieldTypeSet(CacheTypeRanked, 100))
|
|
err := client.SyncSchema(schema)
|
|
if err != nil {
|
|
t.Fatalf("syncing schema: %v", err)
|
|
}
|
|
defer func() {
|
|
err := client.DeleteIndex(idx)
|
|
if err != nil {
|
|
t.Logf("problem cleaning up from test: %v", err)
|
|
}
|
|
}()
|
|
|
|
b, err := NewBatch(client, 3, idx, fields)
|
|
if err != nil {
|
|
t.Fatalf("creating new batch: %v", err)
|
|
}
|
|
|
|
rowmap := map[string]uint64{
|
|
"c": 9,
|
|
"d": 10,
|
|
"f": 13,
|
|
}
|
|
b.rowTranslations["strslice"] = make(map[string]agedTranslation)
|
|
for k, id := range rowmap {
|
|
b.rowTranslations["strslice"][k] = agedTranslation{
|
|
id: id,
|
|
}
|
|
}
|
|
|
|
r := Row{Values: make([]interface{}, len(fields))}
|
|
r.ID = uint64(0)
|
|
r.Values[0] = []string{"a"}
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("adding to batch: %v", err)
|
|
}
|
|
if got := b.toTranslateSets["strslice"]["a"]; !reflect.DeepEqual(got, []int{0}) {
|
|
t.Fatalf("expected []int{0}, got: %v", got)
|
|
}
|
|
|
|
r.ID = uint64(1)
|
|
r.Values[0] = []string{"a", "b", "c"}
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("adding to batch: %v", err)
|
|
}
|
|
if got := b.toTranslateSets["strslice"]["a"]; !reflect.DeepEqual(got, []int{0, 1}) {
|
|
t.Fatalf("expected []int{0,1}, got: %v", got)
|
|
}
|
|
if got := b.toTranslateSets["strslice"]["b"]; !reflect.DeepEqual(got, []int{1}) {
|
|
t.Fatalf("expected []int{1}, got: %v", got)
|
|
}
|
|
if got, ok := b.toTranslateSets["strslice"]["c"]; ok {
|
|
t.Fatalf("should be nothing at c, got: %v", got)
|
|
}
|
|
if got := b.rowIDSets["strslice"][1]; !reflect.DeepEqual(got, []uint64{9}) {
|
|
t.Fatalf("expected c to map to rowID 9 but got %v", got)
|
|
}
|
|
|
|
r.ID = uint64(2)
|
|
r.Values[0] = []string{"d", "e", "f"}
|
|
err = b.Add(r)
|
|
if err != ErrBatchNowFull {
|
|
t.Fatalf("adding to batch: %v", err)
|
|
}
|
|
if got, ok := b.toTranslateSets["strslice"]["d"]; ok {
|
|
t.Fatalf("should be nothing at d, got: %v", got)
|
|
}
|
|
if got, ok := b.toTranslateSets["strslice"]["f"]; ok {
|
|
t.Fatalf("should be nothing at f, got: %v", got)
|
|
}
|
|
if got := b.toTranslateSets["strslice"]["e"]; !reflect.DeepEqual(got, []int{2}) {
|
|
t.Fatalf("expected []int{2}, got: %v", got)
|
|
}
|
|
if got := b.rowIDSets["strslice"][2]; !reflect.DeepEqual(got, []uint64{10, 13}) {
|
|
t.Fatalf("expected c to map to rowID 9 but got %v", got)
|
|
}
|
|
|
|
err = b.doTranslation()
|
|
if err != nil {
|
|
t.Fatalf("translating: %v", err)
|
|
}
|
|
|
|
if got0 := b.rowIDSets["strslice"][0]; len(got0) != 1 {
|
|
t.Errorf("after translation, rec 0, wrong len: %v", got0)
|
|
} else if got1 := b.rowIDSets["strslice"][1]; len(got1) != 3 || got1[0] != 9 || (got1[1] != got0[0] && got1[2] != got0[0]) {
|
|
t.Errorf("after translation, rec 1: %v, rec 0: %v", got1, got0)
|
|
} else if got2 := b.rowIDSets["strslice"][2]; len(got2) != 3 || got2[0] != 10 || got2[1] != 13 || got2[2] == got1[2] || got2[2] == got0[0] {
|
|
t.Errorf("after translation, rec 2: %v", got2)
|
|
}
|
|
|
|
frags, clearFrags, err := b.makeFragments(make(fragments), make(fragments))
|
|
if err != nil {
|
|
t.Errorf("making fragments: %v", err)
|
|
}
|
|
|
|
err = b.doImport(frags, clearFrags)
|
|
if err != nil {
|
|
t.Fatalf("doing import: %v", err)
|
|
}
|
|
|
|
resp, err := client.Query(idx.BatchQuery(fields[0].Row("a")))
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
result := resp.Result()
|
|
if !reflect.DeepEqual(result.Row().Columns, []uint64{0, 1}) {
|
|
t.Fatalf("expected a to be [0,1], got %v", result.Row().Columns)
|
|
}
|
|
}
|
|
|
|
func testSingleClearBatchRegression(t *testing.T, c *test.Cluster, client *Client) {
|
|
schema := NewSchema()
|
|
idx := schema.Index("test-single-clear-batch-regression")
|
|
numFields := 1
|
|
fields := make([]*Field, numFields)
|
|
fields[0] = idx.Field("zero", OptFieldKeys(true))
|
|
|
|
err := client.SyncSchema(schema)
|
|
if err != nil {
|
|
t.Fatalf("syncing schema: %v", err)
|
|
}
|
|
defer func() {
|
|
err := client.DeleteIndex(idx)
|
|
if err != nil {
|
|
t.Logf("problem cleaning up from test: %v", err)
|
|
}
|
|
}()
|
|
|
|
_, err = client.Query(fields[0].Set("row1", 1))
|
|
if err != nil {
|
|
t.Fatalf("setting bit: %v", err)
|
|
}
|
|
|
|
b, err := NewBatch(client, 1, idx, fields)
|
|
if err != nil {
|
|
t.Fatalf("getting new batch: %v", err)
|
|
}
|
|
r := Row{ID: uint64(1), Values: make([]interface{}, numFields), Clears: make(map[int]interface{})}
|
|
r.Values[0] = nil
|
|
r.Clears[0] = "row1"
|
|
err = b.Add(r)
|
|
if err != ErrBatchNowFull {
|
|
t.Fatalf("wrong error from batch add: %v", err)
|
|
}
|
|
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("error importing: %v", err)
|
|
}
|
|
|
|
resp, err := client.Query(fields[0].Row("row1"))
|
|
if err != nil {
|
|
t.Fatalf("error querying: %v", err)
|
|
}
|
|
result := resp.Results()[0].Row().Columns
|
|
if len(result) != 0 {
|
|
t.Fatalf("unexpected values in row: result %+v", result)
|
|
}
|
|
|
|
}
|
|
|
|
func testBatches(t *testing.T, c *test.Cluster, client *Client) {
|
|
schema := NewSchema()
|
|
idx := schema.Index("test-batches")
|
|
numFields := 5
|
|
fields := make([]*Field, numFields)
|
|
fields[0] = idx.Field("zero", OptFieldKeys(true))
|
|
fields[1] = idx.Field("one", OptFieldKeys(true))
|
|
fields[2] = idx.Field("two", OptFieldKeys(true))
|
|
fields[3] = idx.Field("three", OptFieldTypeInt())
|
|
fields[4] = idx.Field("four", OptFieldTypeTime(TimeQuantumYearMonthDay))
|
|
err := client.SyncSchema(schema)
|
|
if err != nil {
|
|
t.Fatalf("syncing schema: %v", err)
|
|
}
|
|
defer func() {
|
|
err := client.DeleteIndex(idx)
|
|
if err != nil {
|
|
t.Logf("problem cleaning up from test: %v", err)
|
|
}
|
|
}()
|
|
b, err := NewBatch(client, 10, idx, fields)
|
|
if err != nil {
|
|
t.Fatalf("getting new batch: %v", err)
|
|
}
|
|
r := Row{Values: make([]interface{}, numFields), Clears: make(map[int]interface{})}
|
|
r.Time.Set(time.Date(2019, time.January, 2, 15, 45, 0, 0, time.UTC))
|
|
|
|
for i := 0; i < 9; i++ {
|
|
r.ID = uint64(i)
|
|
if i%2 == 0 {
|
|
r.Values[0] = "a"
|
|
r.Values[1] = "b"
|
|
r.Values[2] = "c"
|
|
r.Values[3] = int64(99)
|
|
r.Values[4] = uint64(1)
|
|
r.Time.SetMonth("01")
|
|
} else {
|
|
r.Values[0] = "x"
|
|
r.Values[1] = "y"
|
|
r.Values[2] = "z"
|
|
r.Values[3] = int64(-10)
|
|
r.Values[4] = uint64(1)
|
|
r.Time.SetMonth("02")
|
|
}
|
|
if i == 8 {
|
|
r.Values[0] = nil
|
|
r.Clears[1] = uint64(97)
|
|
r.Clears[2] = "c"
|
|
r.Values[3] = nil
|
|
r.Values[4] = nil
|
|
}
|
|
err := b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err adding record: %v", err)
|
|
}
|
|
|
|
}
|
|
|
|
if len(b.toTranslate[0]) != 2 {
|
|
t.Fatalf("wrong number of keys in toTranslate[0]")
|
|
}
|
|
for k, ints := range b.toTranslate[0] {
|
|
if k == "a" {
|
|
if !reflect.DeepEqual(ints, []int{0, 2, 4, 6}) {
|
|
t.Fatalf("wrong ints for key a in field zero: %v", ints)
|
|
}
|
|
} else if k == "x" {
|
|
if !reflect.DeepEqual(ints, []int{1, 3, 5, 7}) {
|
|
t.Fatalf("wrong ints for key x in field zero: %v", ints)
|
|
}
|
|
|
|
} else {
|
|
t.Fatalf("unexpected key %s", k)
|
|
}
|
|
}
|
|
if !reflect.DeepEqual(b.toTranslateClear, map[int]map[string][]int{2: {"c": {8}}}) {
|
|
t.Errorf("unexpected toTranslateClear: %+v", b.toTranslateClear)
|
|
}
|
|
if !reflect.DeepEqual(b.clearRowIDs, map[int]map[int]uint64{1: {8: 97}, 2: {}}) {
|
|
t.Errorf("unexpected clearRowIDs: %+v", b.clearRowIDs)
|
|
}
|
|
|
|
if !reflect.DeepEqual(b.values["three"], []int64{99, -10, 99, -10, 99, -10, 99, -10, 0}) {
|
|
t.Fatalf("unexpected values: %v", b.values["three"])
|
|
}
|
|
if !reflect.DeepEqual(b.nullIndices["three"], []uint64{8}) {
|
|
t.Fatalf("unexpected nullIndices: %v", b.nullIndices["three"])
|
|
}
|
|
|
|
if len(b.toTranslate[1]) != 2 {
|
|
t.Fatalf("wrong number of keys in toTranslate[1]")
|
|
}
|
|
for k, ints := range b.toTranslate[1] {
|
|
if k == "b" {
|
|
if !reflect.DeepEqual(ints, []int{0, 2, 4, 6, 8}) {
|
|
t.Fatalf("wrong ints for key b in field one: %v", ints)
|
|
}
|
|
} else if k == "y" {
|
|
if !reflect.DeepEqual(ints, []int{1, 3, 5, 7}) {
|
|
t.Fatalf("wrong ints for key y in field one: %v", ints)
|
|
}
|
|
|
|
} else {
|
|
t.Fatalf("unexpected key %s", k)
|
|
}
|
|
}
|
|
|
|
if len(b.toTranslate[2]) != 2 {
|
|
t.Fatalf("wrong number of keys in toTranslate[2]")
|
|
}
|
|
for k, ints := range b.toTranslate[2] {
|
|
if k == "c" {
|
|
if !reflect.DeepEqual(ints, []int{0, 2, 4, 6, 8}) {
|
|
t.Fatalf("wrong ints for key c in field two: %v", ints)
|
|
}
|
|
} else if k == "z" {
|
|
if !reflect.DeepEqual(ints, []int{1, 3, 5, 7}) {
|
|
t.Fatalf("wrong ints for key z in field two: %v", ints)
|
|
}
|
|
|
|
} else {
|
|
t.Fatalf("unexpected key %s", k)
|
|
}
|
|
}
|
|
|
|
err = b.Add(r)
|
|
if err != ErrBatchNowFull {
|
|
t.Fatalf("should have gotten full batch error, but got %v", err)
|
|
}
|
|
|
|
err = b.Add(r)
|
|
if err != ErrBatchAlreadyFull {
|
|
t.Fatalf("should have gotten already full batch error, but got %v", err)
|
|
}
|
|
|
|
if !reflect.DeepEqual(b.values["three"], []int64{99, -10, 99, -10, 99, -10, 99, -10, 0, 0}) {
|
|
t.Fatalf("unexpected values: %v", b.values["three"])
|
|
}
|
|
|
|
err = b.doTranslation()
|
|
if err != nil {
|
|
t.Fatalf("doing translation: %v", err)
|
|
}
|
|
|
|
for fidx, rowIDs := range b.rowIDs {
|
|
// we don't know which key will get translated first, but we do know the pattern
|
|
if fidx == 0 {
|
|
if !reflect.DeepEqual(rowIDs, []uint64{1, 2, 1, 2, 1, 2, 1, 2, nilSentinel, nilSentinel}) &&
|
|
!reflect.DeepEqual(rowIDs, []uint64{2, 1, 2, 1, 2, 1, 2, 1, nilSentinel, nilSentinel}) {
|
|
t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs)
|
|
}
|
|
|
|
} else if fidx == 4 {
|
|
if !reflect.DeepEqual(rowIDs, []uint64{1, 1, 1, 1, 1, 1, 1, 1, nilSentinel, nilSentinel}) {
|
|
t.Fatalf("unexpected rowids for time field")
|
|
}
|
|
} else if fidx == 3 {
|
|
if len(rowIDs) != 0 {
|
|
t.Fatalf("expected no rowIDs for int field, but got: %v", rowIDs)
|
|
}
|
|
} else {
|
|
if !reflect.DeepEqual(rowIDs, []uint64{1, 2, 1, 2, 1, 2, 1, 2, 1, nilSentinel}) && !reflect.DeepEqual(rowIDs, []uint64{2, 1, 2, 1, 2, 1, 2, 1, 2, nilSentinel}) {
|
|
t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs)
|
|
}
|
|
}
|
|
}
|
|
|
|
if !reflect.DeepEqual(b.clearRowIDs[1], map[int]uint64{8: 97}) {
|
|
t.Errorf("unexpected clearRowIDs after translation: %+v", b.clearRowIDs[1])
|
|
}
|
|
if !reflect.DeepEqual(b.clearRowIDs[2], map[int]uint64{8: 2}) && !reflect.DeepEqual(b.clearRowIDs[2], map[int]uint64{8: 1}) {
|
|
t.Errorf("unexpected clearRowIDs: after translation%+v", b.clearRowIDs[2])
|
|
}
|
|
|
|
frags, clearFrags, err := b.makeFragments(make(fragments), make(fragments))
|
|
if err != nil {
|
|
t.Errorf("making fragments: %v", err)
|
|
}
|
|
|
|
err = b.doImport(frags, clearFrags)
|
|
if err != nil {
|
|
t.Fatalf("doing import: %v", err)
|
|
}
|
|
|
|
b.reset()
|
|
|
|
for i := 9; i < 19; i++ {
|
|
r.ID = uint64(i)
|
|
if i%2 == 0 {
|
|
r.Values[0] = "a"
|
|
r.Values[1] = "b"
|
|
r.Values[2] = "c"
|
|
r.Values[3] = int64(99)
|
|
r.Values[4] = uint64(1)
|
|
} else {
|
|
r.Values[0] = "x"
|
|
r.Values[1] = "y"
|
|
r.Values[2] = "z"
|
|
r.Values[3] = int64(-10)
|
|
r.Values[4] = uint64(2)
|
|
}
|
|
err := b.Add(r)
|
|
if i != 18 && err != nil {
|
|
t.Fatalf("unexpected err adding record: %v", err)
|
|
}
|
|
if i == 18 && err != ErrBatchNowFull {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
}
|
|
|
|
// should do nothing
|
|
err = b.doTranslation()
|
|
if err != nil {
|
|
t.Fatalf("doing translation: %v", err)
|
|
}
|
|
|
|
frags, clearFrags, err = b.makeFragments(make(fragments), make(fragments))
|
|
if err != nil {
|
|
t.Errorf("making fragments: %v", err)
|
|
}
|
|
|
|
err = b.doImport(frags, clearFrags)
|
|
if err != nil {
|
|
t.Fatalf("doing import: %v", err)
|
|
}
|
|
|
|
for fidx, rowIDs := range b.rowIDs {
|
|
if fidx == 3 {
|
|
if len(rowIDs) != 0 {
|
|
t.Fatalf("expected no rowIDs for int field, but got: %v", rowIDs)
|
|
}
|
|
continue
|
|
}
|
|
// we don't know which key will get translated first, but we do know the pattern
|
|
if !reflect.DeepEqual(rowIDs, []uint64{1, 2, 1, 2, 1, 2, 1, 2, 1, 2}) && !reflect.DeepEqual(rowIDs, []uint64{2, 1, 2, 1, 2, 1, 2, 1, 2, 1}) {
|
|
t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs)
|
|
}
|
|
}
|
|
|
|
b.reset()
|
|
|
|
for i := 19; i < 29; i++ {
|
|
r.ID = uint64(i)
|
|
if i%2 == 0 {
|
|
r.Values[0] = "d"
|
|
r.Values[1] = "e"
|
|
r.Values[2] = "f"
|
|
r.Values[3] = int64(100)
|
|
r.Values[4] = uint64(3)
|
|
} else {
|
|
r.Values[0] = "u"
|
|
r.Values[1] = "v"
|
|
r.Values[2] = "w"
|
|
r.Values[3] = int64(0)
|
|
r.Values[4] = uint64(4)
|
|
}
|
|
err := b.Add(r)
|
|
if i != 28 && err != nil {
|
|
t.Fatalf("unexpected err adding record: %v", err)
|
|
}
|
|
if i == 28 && err != ErrBatchNowFull {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
}
|
|
|
|
err = b.doTranslation()
|
|
if err != nil {
|
|
t.Fatalf("doing translation: %v", err)
|
|
}
|
|
|
|
frags, clearFrags, err = b.makeFragments(make(fragments), make(fragments))
|
|
if err != nil {
|
|
t.Errorf("making fragments: %v", err)
|
|
}
|
|
|
|
err = b.doImport(frags, clearFrags)
|
|
if err != nil {
|
|
t.Fatalf("doing import: %v", err)
|
|
}
|
|
|
|
for fidx, rowIDs := range b.rowIDs {
|
|
// we don't know which key will get translated first, but we do know the pattern
|
|
if fidx == 3 {
|
|
if len(rowIDs) != 0 {
|
|
t.Fatalf("expected no rowIDs for int field, but got: %v", rowIDs)
|
|
}
|
|
continue
|
|
}
|
|
if !reflect.DeepEqual(rowIDs, []uint64{3, 4, 3, 4, 3, 4, 3, 4, 3, 4}) && !reflect.DeepEqual(rowIDs, []uint64{4, 3, 4, 3, 4, 3, 4, 3, 4, 3}) {
|
|
t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs)
|
|
}
|
|
}
|
|
|
|
frags, _, err = b.makeFragments(make(fragments), make(fragments))
|
|
if err != nil {
|
|
t.Fatalf("making fragments: %v", err)
|
|
}
|
|
|
|
var n int
|
|
for key := range frags {
|
|
if key.shard == 0 {
|
|
n++
|
|
}
|
|
}
|
|
if n != 5 { // zero, one, two, four (three is an int field so not in fragments) + _exists
|
|
t.Fatalf("there should be 5 views, but have %d", n)
|
|
}
|
|
|
|
resp, err := client.Query(idx.BatchQuery(fields[0].Row("a"),
|
|
fields[1].Row("b"),
|
|
fields[2].Row("c"),
|
|
fields[3].Equals(99)))
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
|
|
results := resp.Results()
|
|
for _, j := range []int{0, 2, 3} {
|
|
cols := results[j].Row().Columns
|
|
if !reflect.DeepEqual(cols, []uint64{0, 2, 4, 6, 10, 12, 14, 16, 18}) {
|
|
t.Fatalf("unexpected columns for a: %v", cols)
|
|
}
|
|
}
|
|
res := results[1]
|
|
|
|
if cols := res.Row().Columns; !reflect.DeepEqual(cols, []uint64{0, 2, 4, 6, 8, 10, 12, 14, 16, 18}) {
|
|
t.Fatalf("unexpected columns for field 1 row b: %v", cols)
|
|
}
|
|
|
|
resp, err = client.Query(idx.BatchQuery(fields[0].Row("d"),
|
|
fields[1].Row("e"),
|
|
fields[2].Row("f")))
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
|
|
results = resp.Results()
|
|
for _, res := range results {
|
|
cols := res.Row().Columns
|
|
if !reflect.DeepEqual(cols, []uint64{20, 22, 24, 26, 28}) {
|
|
t.Fatalf("unexpected columns: %v", cols)
|
|
}
|
|
}
|
|
|
|
resp, err = client.Query(idx.BatchQuery(fields[3].GT(-11),
|
|
fields[3].Equals(0),
|
|
fields[3].Equals(100),
|
|
fields[4].Range(1, time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC), time.Date(2019, time.January, 29, 0, 0, 0, 0, time.UTC)),
|
|
fields[4].Range(1, time.Date(2019, time.February, 1, 0, 0, 0, 0, time.UTC), time.Date(2019, time.February, 29, 0, 0, 0, 0, time.UTC))))
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
results = resp.Results()
|
|
|
|
if cols := results[0].Row().Columns; !reflect.DeepEqual(cols, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}) {
|
|
t.Fatalf("all columns (but 8) should be greater than -11, but got: %v", cols)
|
|
}
|
|
|
|
if cols := results[1].Row().Columns; !reflect.DeepEqual(cols, []uint64{19, 21, 23, 25, 27}) {
|
|
t.Fatalf("wrong cols for ==0: %v", cols)
|
|
}
|
|
|
|
if cols := results[2].Row().Columns; !reflect.DeepEqual(cols, []uint64{20, 22, 24, 26, 28}) {
|
|
t.Fatalf("wrong cols for ==100: %v", cols)
|
|
}
|
|
|
|
cols := results[3].Row().Columns
|
|
exp := []uint64{0, 2, 4, 6, 10, 12, 14, 16, 18}
|
|
if !reflect.DeepEqual(cols, exp) {
|
|
t.Fatalf("wrong cols for January: got/want\n%v\n%v", cols, exp)
|
|
}
|
|
|
|
cols = results[4].Row().Columns
|
|
exp = []uint64{1, 3, 5, 7}
|
|
if !reflect.DeepEqual(cols, exp) {
|
|
t.Fatalf("wrong cols for January: got/want\n%v\n%v", cols, exp)
|
|
}
|
|
|
|
b.reset()
|
|
r.ID = uint64(0)
|
|
r.Values[0] = "x"
|
|
r.Values[1] = "b"
|
|
r.Clears[0] = "a"
|
|
r.Clears[1] = "b" // b should get cleared
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("adding with clears: %v", err)
|
|
}
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("importing w/clears: %v", err)
|
|
}
|
|
resp, err = client.Query(idx.BatchQuery(
|
|
fields[0].Row("a"),
|
|
fields[0].Row("x"),
|
|
fields[1].Row("b"),
|
|
))
|
|
if err != nil {
|
|
t.Fatalf("querying after clears: %v", err)
|
|
}
|
|
if arow := resp.Results()[0].Row().Columns; arow[0] == 0 {
|
|
t.Errorf("shouldn't have id 0 in row a after clearing! %v", arow)
|
|
}
|
|
if xrow := resp.Results()[1].Row().Columns; xrow[0] != 0 {
|
|
t.Errorf("should have id 0 in row x after setting %v", xrow)
|
|
}
|
|
if brow := resp.Results()[2].Row().Columns; brow[0] == 0 {
|
|
t.Errorf("shouldn't have id 0 in row b after clearing! %v", brow)
|
|
}
|
|
|
|
// TODO test importing across multiple shards
|
|
}
|
|
|
|
func testBatchesStringIDs(t *testing.T, c *test.Cluster, client *Client) {
|
|
schema := NewSchema()
|
|
idx := schema.Index("batches-strings-ids", OptIndexKeys(true))
|
|
fields := make([]*Field, 3)
|
|
fields[0] = idx.Field("zero", OptFieldKeys(true))
|
|
fields[1] = idx.Field("one", OptFieldTypeMutex(CacheTypeNone, 0), OptFieldKeys(true))
|
|
fields[2] = idx.Field("two", OptFieldTypeTime("YMDH"), OptFieldKeys(true))
|
|
err := client.SyncSchema(schema)
|
|
if err != nil {
|
|
t.Fatalf("syncing schema: %v", err)
|
|
}
|
|
defer func() {
|
|
err := client.DeleteIndex(idx)
|
|
if err != nil {
|
|
t.Logf("problem cleaning up from test: %v", err)
|
|
}
|
|
}()
|
|
|
|
b, err := NewBatch(client, 3, idx, fields)
|
|
if err != nil {
|
|
t.Fatalf("getting new batch: %v", err)
|
|
}
|
|
|
|
r := Row{Values: make([]interface{}, 3)}
|
|
r.Time.Set(time.Date(2019, time.January, 2, 15, 45, 0, 0, time.UTC))
|
|
|
|
for i := 0; i < 3; i++ {
|
|
r.ID = strconv.Itoa(i)
|
|
if i%2 == 0 {
|
|
r.Values[0] = "a"
|
|
r.Values[1] = "b"
|
|
r.Values[2] = "c"
|
|
r.Time.SetMonth("01")
|
|
} else {
|
|
r.Values[0] = "x"
|
|
r.Values[1] = "y"
|
|
r.Values[2] = "z"
|
|
r.Time.SetMonth("02")
|
|
}
|
|
err := b.Add(r)
|
|
if err != nil && err != ErrBatchNowFull {
|
|
t.Fatalf("unexpected err adding record: %v", err)
|
|
}
|
|
}
|
|
|
|
if len(b.toTranslateID) != 3 {
|
|
t.Fatalf("id translation table unexpected size: %v", b.toTranslateID)
|
|
}
|
|
for i, k := range b.toTranslateID {
|
|
if ik, err := strconv.Atoi(k); err != nil || ik != i {
|
|
t.Errorf("unexpected toTranslateID key %s at index %d", k, i)
|
|
}
|
|
}
|
|
|
|
err = b.doTranslation()
|
|
if err != nil {
|
|
t.Fatalf("translating: %v", err)
|
|
}
|
|
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("importing: %v", err)
|
|
}
|
|
|
|
resp, err := client.Query(idx.BatchQuery(fields[0].Row("a"), fields[0].Row("x"), fields[1].Row("b"), fields[1].Row("y"), fields[2].Row("c"), fields[2].Row("z")))
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
|
|
results := resp.Results()
|
|
for i, res := range results {
|
|
cols := res.Row().Keys
|
|
if i%2 == 0 && !reflect.DeepEqual(cols, []string{"0", "2"}) && !reflect.DeepEqual(cols, []string{"2", "0"}) {
|
|
t.Fatalf("unexpected columns: %v", cols)
|
|
}
|
|
if i%2 == 1 && !reflect.DeepEqual(cols, []string{"1"}) {
|
|
t.Fatalf("unexpected columns: %v", cols)
|
|
}
|
|
}
|
|
|
|
b.reset()
|
|
|
|
r.ID = "1"
|
|
r.Values[0] = "a"
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err adding record: %v", err)
|
|
}
|
|
|
|
r.ID = "3"
|
|
r.Values[0] = "z"
|
|
err = b.Add(r)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err adding record: %v", err)
|
|
}
|
|
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("importing: %v", err)
|
|
}
|
|
|
|
resp, err = client.Query(idx.BatchQuery(fields[0].Row("a"), fields[0].Row("z")))
|
|
if err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
}
|
|
|
|
results = resp.Results()
|
|
for i, res := range results {
|
|
cols := res.Row().Keys
|
|
if err := isPermutationOf(cols, []string{"0", "1", "2"}); i == 0 && err != nil {
|
|
t.Fatalf("unexpected columns: %v: %v", cols, err)
|
|
}
|
|
if i == 1 && !reflect.DeepEqual(cols, []string{"3"}) {
|
|
t.Fatalf("unexpected columns: %v", cols)
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func isPermutationOf(one, two []string) error {
|
|
if len(one) != len(two) {
|
|
return errors.Errorf("different lengths %d and %d", len(one), len(two))
|
|
}
|
|
outer:
|
|
for _, vOne := range one {
|
|
for j, vTwo := range two {
|
|
if vOne == vTwo {
|
|
two = append(two[:j], two[j+1:]...)
|
|
continue outer
|
|
}
|
|
}
|
|
return errors.Errorf("%s in one but not two", vOne)
|
|
}
|
|
if len(two) != 0 {
|
|
return errors.Errorf("vals in two but not one: %v", two)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func TestQuantizedTime(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
time time.Time
|
|
year string
|
|
month string
|
|
day string
|
|
hour string
|
|
quantum TimeQuantum
|
|
reset bool
|
|
exp []string
|
|
expErr string
|
|
}{
|
|
{
|
|
name: "no time quantum",
|
|
expErr: "",
|
|
},
|
|
{
|
|
name: "no time quantum with data",
|
|
year: "2017",
|
|
exp: []string{},
|
|
expErr: "",
|
|
},
|
|
{
|
|
name: "no data",
|
|
quantum: TimeQuantumYear,
|
|
exp: nil,
|
|
expErr: "",
|
|
},
|
|
{
|
|
name: "timestamp",
|
|
time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
|
|
quantum: "YMDH",
|
|
exp: []string{"2013", "201310", "20131016", "2013101617"},
|
|
},
|
|
{
|
|
name: "timestamp-less-granular",
|
|
time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
|
|
quantum: "YM",
|
|
exp: []string{"2013", "201310"},
|
|
},
|
|
{
|
|
name: "timestamp-mid-granular",
|
|
time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
|
|
quantum: "MD",
|
|
exp: []string{"201310", "20131016"},
|
|
},
|
|
{
|
|
name: "justyear",
|
|
year: "2013",
|
|
quantum: "Y",
|
|
exp: []string{"2013"},
|
|
},
|
|
{
|
|
name: "justyear-wantmonth",
|
|
year: "2013",
|
|
quantum: "YM",
|
|
expErr: "no data set for month",
|
|
},
|
|
{
|
|
name: "timestamp-changeyear",
|
|
time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
|
|
year: "2019",
|
|
quantum: "YMDH",
|
|
exp: []string{"2019", "201910", "20191016", "2019101617"},
|
|
},
|
|
{
|
|
name: "yearmonthdayhour",
|
|
year: "2013",
|
|
month: "10",
|
|
day: "16",
|
|
hour: "17",
|
|
quantum: "YMDH",
|
|
exp: []string{"2013", "201310", "20131016", "2013101617"},
|
|
},
|
|
{
|
|
name: "timestamp-changehour",
|
|
time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
|
|
hour: "05",
|
|
quantum: "MDH",
|
|
exp: []string{"201310", "20131016", "2013101605"},
|
|
},
|
|
{
|
|
name: "timestamp",
|
|
time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
|
|
quantum: "YMDH",
|
|
reset: true,
|
|
exp: nil,
|
|
},
|
|
}
|
|
|
|
for i, test := range cases {
|
|
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
|
|
tq := QuantizedTime{}
|
|
var zt time.Time
|
|
if zt != test.time {
|
|
tq.Set(test.time)
|
|
}
|
|
if test.year != "" {
|
|
tq.SetYear(test.year)
|
|
}
|
|
if test.month != "" {
|
|
tq.SetMonth(test.month)
|
|
}
|
|
if test.day != "" {
|
|
tq.SetDay(test.day)
|
|
}
|
|
if test.hour != "" {
|
|
tq.SetHour(test.hour)
|
|
}
|
|
if test.reset {
|
|
tq.Reset()
|
|
}
|
|
|
|
views, err := tq.views(test.quantum)
|
|
if !reflect.DeepEqual(views, test.exp) {
|
|
t.Errorf("unexpected views, got/want:\n%v\n%v\n", views, test.exp)
|
|
}
|
|
if (err != nil && err.Error() != test.expErr) || (err == nil && test.expErr != "") {
|
|
t.Errorf("unexpected error, got/want:\n%v\n%s\n", err, test.expErr)
|
|
}
|
|
})
|
|
}
|
|
|
|
}
|
|
|
|
func testBatchStaleness(t *testing.T, c *test.Cluster, client *Client) {
|
|
schema := NewSchema()
|
|
idx := schema.Index("test-batch-staleness")
|
|
field := idx.Field("anint", OptFieldTypeInt())
|
|
err := client.SyncSchema(schema)
|
|
if err != nil {
|
|
t.Fatalf("syncing schema: %v", err)
|
|
}
|
|
defer func() {
|
|
err := client.DeleteIndex(idx)
|
|
if err != nil {
|
|
t.Logf("problem cleaning up from test: %v", err)
|
|
}
|
|
}()
|
|
|
|
b, err := NewBatch(client, 3, idx, []*Field{field}, OptMaxStaleness(time.Millisecond))
|
|
if err != nil {
|
|
t.Fatalf("getting batch: %v", err)
|
|
}
|
|
|
|
r := Row{ID: uint64(0), Values: []interface{}{int64(0)}}
|
|
err = b.Add(r)
|
|
if err != nil && err != ErrBatchNowFull {
|
|
t.Fatalf("adding to batch: %v", err)
|
|
}
|
|
|
|
// sleep so batch becomes stale
|
|
time.Sleep(time.Millisecond)
|
|
|
|
r = Row{ID: uint64(1), Values: []interface{}{int64(0)}}
|
|
err = b.Add(r)
|
|
if err != ErrBatchNowStale {
|
|
t.Fatal("batch expected to be stale")
|
|
}
|
|
}
|
|
|
|
func testImportBatchMultipleInts(t *testing.T, c *test.Cluster, client *Client) {
|
|
schema := NewSchema()
|
|
idx := schema.Index("test-import-batch-multi-int")
|
|
field := idx.Field("anint", OptFieldTypeInt())
|
|
err := client.SyncSchema(schema)
|
|
if err != nil {
|
|
t.Fatalf("syncing schema: %v", err)
|
|
}
|
|
|
|
b, err := NewBatch(client, 6, idx, []*Field{field}, OptUseShardTransactionalEndpoint(true))
|
|
if err != nil {
|
|
t.Fatalf("getting batch: %v", err)
|
|
}
|
|
|
|
r := Row{Values: make([]interface{}, 1)}
|
|
|
|
vals := []int64{16, 8, 32, 1, 2, 4}
|
|
for i := uint64(0); i < 6; i++ {
|
|
r.ID = uint64(1)
|
|
r.Values[0] = vals[i]
|
|
err := b.Add(r)
|
|
if err != nil && err != ErrBatchNowFull {
|
|
t.Fatalf("adding to batch: %v", err)
|
|
}
|
|
}
|
|
err = b.Import()
|
|
if err != nil {
|
|
t.Fatalf("importing: %v", err)
|
|
}
|
|
|
|
if resp, err := client.Query(field.Equals(4)); err != nil {
|
|
t.Fatalf("querying: %v", err)
|
|
} else if res := resp.Results()[0].Row().Columns; len(res) != 1 || res[0] != 1 {
|
|
t.Fatalf("unepxected result: %v", res)
|
|
}
|
|
|
|
}
|