featurebase/executor_internal_test.go
Seebs cf97a0dcb8 overhaul: switch over to using QueryContext
We switch everything to use QueryContext/QueryRead/etc instead
of Qcx/Tx.

We drop the short_txkey subpackage (it's now handled by either
keys or querycontext).

We drop all the dbshard stuff, and all the tx/txfactory stuff.

We remove all the things that related to the old "Block" concept,
which was mostly used by the anti-entropy code, but had one
fragmentary usage left in the ImportRoaringOverwrite case of
ImportRoaring. That's replaced by using a rewriter that deletes
all bits (not just bits in specific columns) from an existing
thing, but writes in new bits. Actually we could probably do that
better with a custom "eradicate-rewriter" that doesn't try to
be clever, and just eliminates things.

This includes a number of minor bug fixes that were
exposed by getting the testing to work. For example:
* When checking whether an operation "requires write", we
  now consider a Delete a kind of a Write, because it is.
* Several tests were relying on the fact that writes through
  Qcx were being committed whether or not the Qcx was ever
  told to finish. With QueryContext, you actually have to
  reach a Commit() or the writes don't happen (except for
  special cases in Delete).
* Replaced a lot of panics with t.Fatalf in tests.

There's also some minor staticcheck fixes, like deleting the
unused "db" member of a boltdb transaction wrapper.
2023-01-11 12:57:56 -06:00

600 lines
16 KiB
Go

// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"context"
"encoding/json"
"fmt"
"reflect"
"sort"
"strconv"
"testing"
"time"
"github.com/molecula/featurebase/v3/pql"
"github.com/stretchr/testify/require"
)
// AssertEqual checks a given RowIdentifiers against expected values.
func (r *RowIdentifiers) AssertEqual(tb testing.TB, other *RowIdentifiers) {
sort.Slice(r.Rows, func(i, j int) bool { return r.Rows[i] < r.Rows[j] })
sort.Slice(other.Rows, func(i, j int) bool { return other.Rows[i] < other.Rows[j] })
if len(r.Rows) != len(other.Rows) {
tb.Fatalf("row ID mismatch: got %d, expected %d", r.Rows, other.Rows)
}
for i := range r.Rows {
if r.Rows[i] != other.Rows[i] {
tb.Fatalf("row ID mismatch: got %d, expected %d", r.Rows, other.Rows)
}
}
sort.Strings(r.Keys)
sort.Strings(other.Keys)
if len(r.Keys) != len(other.Keys) {
tb.Fatalf("row keys mismatch: got %s, expected %s", r.Keys, other.Keys)
}
for i := range r.Keys {
if r.Keys[i] != other.Keys[i] {
tb.Fatalf("row keys mismatch: got %s, expected %s", r.Keys, other.Keys)
}
}
}
func TestExecutor_TranslateRowsOnBool(t *testing.T) {
holder := newTestHolder(t)
e := &executor{
Holder: holder,
Cluster: NewTestCluster(t, 1),
}
idx, err := e.Holder.CreateIndex("i", "", IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
qcx := holder.MustIndexQueryContext(t, "i")
fb, errb := idx.CreateField("b", "", OptFieldTypeBool())
_, errbk := idx.CreateField("bk", "", OptFieldTypeBool(), OptFieldKeys())
if errb != nil || errbk != nil {
t.Fatalf("creating fields %v, %v", errb, errbk)
}
_, err1 := fb.SetBit(qcx, 1, 1, nil)
_, err2 := fb.SetBit(qcx, 2, 2, nil)
_, err3 := fb.SetBit(qcx, 3, 3, nil)
if err1 != nil || err2 != nil || err3 != nil {
t.Fatalf("setting bit %v, %v, %v", err1, err2, err3)
}
tests := []struct {
pql string
}{
{pql: "Rows(b)"},
{pql: "GroupBy(Rows(b))"},
{pql: "Set(4, b=true)"},
}
for _, test := range tests {
t.Run(test.pql, func(t *testing.T) {
query, err := pql.ParseString(test.pql)
if err != nil {
t.Fatalf("parsing query: %v", err)
}
c := query.Calls[0]
colTranslations, rowTranslations, err := e.preTranslate(context.Background(), "i", c)
if err != nil {
t.Fatalf("pre-translating call: %v", err)
}
_, err = e.translateCall(c, "i", colTranslations, rowTranslations)
if err != nil {
t.Fatalf("translating call: %v", err)
}
})
}
}
func TestFieldRowMarshalJSON(t *testing.T) {
fr := FieldRow{
Field: "blah",
RowID: 0,
RowKey: "ha",
}
b, err := json.Marshal(fr)
if err != nil {
t.Fatalf("marshalling fieldrow: %v", err)
}
if string(b) != `{"field":"blah","rowKey":"ha"}` {
t.Fatalf("unexpected json: %s", b)
}
fr = FieldRow{
Field: "blah",
RowID: 2,
RowKey: "",
}
b, err = json.Marshal(fr)
if err != nil {
t.Fatalf("marshalling fieldrow: %v", err)
}
if string(b) != `{"field":"blah","rowID":2}` {
t.Fatalf("unexpected json: %s", b)
}
}
func TestExecutor_GroupCountCondition(t *testing.T) {
t.Run("satisfiesCondition", func(t *testing.T) {
type condCheck struct {
cond string
exp bool
}
tests := []struct {
groupCount GroupCount
checks []condCheck
}{
{
groupCount: GroupCount{Count: 100},
checks: []condCheck{
{cond: "count == 99", exp: false},
{cond: "count != 99", exp: true},
{cond: "count < 99", exp: false},
{cond: "count <= 99", exp: false},
{cond: "count > 99", exp: true},
{cond: "count >= 99", exp: true},
{cond: "count == 100", exp: true},
{cond: "count != 100", exp: false},
{cond: "count < 100", exp: false},
{cond: "count <= 100", exp: true},
{cond: "count > 100", exp: false},
{cond: "count >= 100", exp: true},
{cond: "count == 101", exp: false},
{cond: "count != 101", exp: true},
{cond: "count < 101", exp: true},
{cond: "count <= 101", exp: true},
{cond: "count > 101", exp: false},
{cond: "count >= 101", exp: false},
{cond: "98 < count < 100", exp: false},
{cond: "98 < count <= 100", exp: true},
{cond: "98 < count < 101", exp: true},
{cond: "100 <= count < 102", exp: true},
{cond: "100 < count < 102", exp: false},
{cond: "98 <= count <= 102", exp: true},
},
},
{
groupCount: GroupCount{Agg: 100},
checks: []condCheck{
{cond: "sum == 99", exp: false},
{cond: "sum != 99", exp: true},
{cond: "sum < 99", exp: false},
{cond: "sum <= 99", exp: false},
{cond: "sum > 99", exp: true},
{cond: "sum >= 99", exp: true},
{cond: "sum == 100", exp: true},
{cond: "sum != 100", exp: false},
{cond: "sum < 100", exp: false},
{cond: "sum <= 100", exp: true},
{cond: "sum > 100", exp: false},
{cond: "sum >= 100", exp: true},
{cond: "sum == 101", exp: false},
{cond: "sum != 101", exp: true},
{cond: "sum < 101", exp: true},
{cond: "sum <= 101", exp: true},
{cond: "sum > 101", exp: false},
{cond: "sum >= 101", exp: false},
{cond: "98 < sum < 100", exp: false},
{cond: "98 < sum <= 100", exp: true},
{cond: "98 < sum < 101", exp: true},
{cond: "100 <= sum < 102", exp: true},
{cond: "100 < sum < 102", exp: false},
{cond: "98 <= sum <= 102", exp: true},
},
},
{
groupCount: GroupCount{Agg: -100},
checks: []condCheck{
{cond: "sum == -99", exp: false},
{cond: "sum != -99", exp: true},
{cond: "sum < -99", exp: true},
{cond: "sum <= -99", exp: true},
{cond: "sum > -99", exp: false},
{cond: "sum >= -99", exp: false},
{cond: "sum == -100", exp: true},
{cond: "sum != -100", exp: false},
{cond: "sum < -100", exp: false},
{cond: "sum <= -100", exp: true},
{cond: "sum > -100", exp: false},
{cond: "sum >= -100", exp: true},
{cond: "sum == -101", exp: false},
{cond: "sum != -101", exp: true},
{cond: "sum < -101", exp: false},
{cond: "sum <= -101", exp: false},
{cond: "sum > -101", exp: true},
{cond: "sum >= -101", exp: true},
{cond: "-100 < sum < -98", exp: false},
{cond: "-100 <= sum < -98", exp: true},
{cond: "-101 < sum < -98", exp: true},
{cond: "-102 < sum <= -100", exp: true},
{cond: "-102 < sum < -100", exp: false},
{cond: "-102 <= sum <= -98", exp: true},
},
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("test (#%d):", i), func(t *testing.T) {
for j, check := range test.checks {
t.Run(fmt.Sprintf("check (#%d):", j), func(t *testing.T) {
query, err := pql.ParseString(fmt.Sprintf("GroupBy(Rows(a), having=Condition(%s))", check.cond))
if err != nil {
t.Fatalf("parsing query: %v", err)
}
c := query.Calls[0]
having := c.Args["having"].(*pql.Call)
var got bool
for subj, cond := range having.Args {
switch subj {
case "count", "sum":
condition, ok := cond.(*pql.Condition)
if !ok {
t.Fatalf("not a valid condition")
}
got = test.groupCount.satisfiesCondition(subj, condition)
}
}
if got != check.exp {
t.Fatalf("expected: %v, but got: %v", check.exp, got)
}
})
}
})
}
})
}
func TestValCountComparisons(t *testing.T) {
tests := []struct {
name string
vc ValCount
other ValCount
expLarger ValCount
expSmaller ValCount
}{
{
name: "zero",
},
{
name: "ints",
vc: ValCount{Val: 10, Count: 1},
other: ValCount{Val: 3, Count: 2},
expLarger: ValCount{Val: 10, Count: 1},
expSmaller: ValCount{Val: 3, Count: 2},
},
{
name: "floats",
vc: ValCount{FloatVal: 10.2, Count: 1},
other: ValCount{FloatVal: 3.4, Count: 2},
expLarger: ValCount{FloatVal: 10.2, Count: 1},
expSmaller: ValCount{FloatVal: 3.4, Count: 2},
},
{
name: "intsEquality",
vc: ValCount{Val: 10, Count: 1},
other: ValCount{Val: 10, Count: 2},
expLarger: ValCount{Val: 10, Count: 3},
expSmaller: ValCount{Val: 10, Count: 3},
},
{
name: "floatsEquality",
vc: ValCount{FloatVal: 10.7, Count: 1},
other: ValCount{FloatVal: 10.7, Count: 2},
expLarger: ValCount{FloatVal: 10.7, Count: 3},
expSmaller: ValCount{FloatVal: 10.7, Count: 3},
},
{
name: "timestampEquality",
vc: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
other: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
expLarger: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 2},
expSmaller: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 2},
},
{
name: "timestamp",
vc: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
other: ValCount{Val: 1587399600, TimestampVal: time.Unix(0, 1587399600*int64(time.Second)), Count: 1},
expLarger: ValCount{Val: 1587399600, TimestampVal: time.Unix(0, 1587399600*int64(time.Second)), Count: 1},
expSmaller: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
},
}
for i, test := range tests {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
gotLarger := test.vc.Larger(test.other)
if gotLarger != test.expLarger {
t.Fatalf("larger failed, expected:\n%+v\ngot:\n%+v", test.expLarger, gotLarger)
}
gotSmaller := test.vc.Smaller(test.other)
if gotSmaller != test.expSmaller {
t.Fatalf("smaller failed, expected:\n%+v\ngot:\n%+v", test.expSmaller, gotSmaller)
}
})
}
}
func TestToNegInt64(t *testing.T) {
tests := []struct {
u64 uint64
i64 int64
overflow bool
}{
{
u64: uint64(1 << 63),
i64: int64(-1 << 63),
},
{
u64: uint64(1<<63) - 1,
i64: int64(-1<<63) + 1,
},
{
u64: uint64(1<<63) + 1,
overflow: true,
},
}
for _, tc := range tests {
val, err := toNegInt64(tc.u64)
if err != nil && !tc.overflow {
t.Fatalf("error: %+v, expected: %+v", err, tc)
}
if val != tc.i64 {
t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
}
}
}
func TestToInt64(t *testing.T) {
tests := []struct {
u64 uint64
i64 int64
overflow bool
}{
{
u64: uint64(1<<63) - 1,
i64: 1<<63 - 1,
},
{
u64: uint64(0),
i64: 0,
},
{
u64: uint64(1 << 63),
overflow: true,
},
{
u64: 1<<64 - 1,
overflow: true,
},
}
for _, tc := range tests {
val, err := toInt64(tc.u64)
if err != nil && !tc.overflow {
t.Fatalf("error: %+v, expected: %+v", err, tc)
}
if val != tc.i64 {
t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
}
}
}
func TestGetSorter(t *testing.T) {
tests := []struct {
sortSpec string
expGCS *groupCountSorter
expErr string
}{
{
sortSpec: "count asc",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: "count asc",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: " count asc",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: " count asc ",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: "count",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{desc}},
},
{
sortSpec: "sum asc",
expGCS: &groupCountSorter{fields: []int{-2}, order: []order{asc}},
},
{
sortSpec: "aggregate asc",
expGCS: &groupCountSorter{fields: []int{-2}, order: []order{asc}},
},
{
sortSpec: "boondoggle asc",
expErr: "sorting is only supported on count, aggregate, or sum, not 'boondoggle'",
},
{
sortSpec: "sum asc, count desc",
expGCS: &groupCountSorter{fields: []int{-2, -1}, order: []order{asc, desc}},
},
{
sortSpec: "count asc, sum desc",
expGCS: &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}},
},
{
sortSpec: " count asc , sum desc ",
expGCS: &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}},
},
{
sortSpec: " count asc , sum desc blah",
expErr: "parsing sort directive: 'sum desc blah': too many elements",
},
{
sortSpec: "count asc, sum fesc",
expErr: "unknown sort direction 'fesc'",
},
{
sortSpec: " , sum fesc",
expErr: "invalid sorting directive: ''",
},
{
// weird and useless, but I guess fine?
sortSpec: "count asc,count asc ",
expGCS: &groupCountSorter{fields: []int{-1, -1}, order: []order{asc, asc}},
},
}
for i, tst := range tests {
t.Run(fmt.Sprintf("%s_%d", tst.sortSpec, i), func(t *testing.T) {
gcs, err := getSorter(tst.sortSpec)
if err != nil {
if tst.expErr == "" {
t.Errorf("unexpected error: %v", err)
return
}
if tst.expErr != err.Error() {
t.Errorf("mismatched errors got: '%v', exp: '%s'", err, tst.expErr)
}
return
}
if !reflect.DeepEqual(gcs, tst.expGCS) {
t.Errorf("exp:\n%+v\ngot:\n%v\n", tst.expGCS, gcs)
}
})
}
}
func TestExecutorSafeCopyDistinctTimestamp(t *testing.T) {
result := DistinctTimestamp{Values: []string{"test", "test"}, Name: "test"}
results := make([]interface{}, 1)
results[0] = result
response := QueryResponse{Results: results, Err: nil, Profile: nil}
copied := safeCopy(response)
if !reflect.DeepEqual(copied.Results, response.Results) {
t.Fatalf("Did not copy results. got %+v, want %+v", copied.Results, response.Results)
}
}
func TestGetScaledInt(t *testing.T) {
_, _, f := newTestField(t, OptFieldTypeTimestamp(time.Now(), "ms"))
// check that fields with type timestamp return the int64 passed in to getScaledInt with nil err
v := time.Now().Unix()
res, err := getScaledInt(f, v)
if err != nil {
t.Errorf("got error %v, expected nil", err)
}
if !reflect.DeepEqual(res, v) {
t.Errorf("expected %v, got %v", v, res)
}
}
func TestDistinctTimestampUnion(t *testing.T) {
cases := []struct {
name string
a DistinctTimestamp
b DistinctTimestamp
expected DistinctTimestamp
}{
{
name: "empty other",
a: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
b: DistinctTimestamp{Name: "a", Values: []string{}},
expected: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
},
{
name: "one more in other",
a: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
b: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c", "d"}},
expected: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c", "d"}},
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
res := test.a.Union(test.b)
allThere := true
for _, val := range res.Values {
here := false
for _, expected := range test.expected.Values {
if val == expected {
here = true
break
}
}
allThere = allThere && here
}
if !allThere {
t.Errorf("expected %v, got %v", test.expected, res)
}
})
}
}
func TestExecutor_DeleteRows(t *testing.T) {
holder := newTestHolder(t)
idx, err := holder.CreateIndex("i", "", IndexOptions{TrackExistence: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
f, err := idx.CreateField("f", "")
if err != nil {
t.Fatalf("creating field: %v", err)
}
qcx := holder.MustIndexQueryContext(t, "i")
if _, err = f.SetBit(qcx, 1, 1, nil); err != nil {
t.Fatalf("setting bit: %v", err)
}
// We rely here on the fact that write Qcx autocommit constantly.
row, err := f.Row(qcx, 1)
if err != nil {
t.Fatalf("failed to read row: %v", err)
}
require.Nil(t, qcx.Commit())
ctx := context.Background()
// make a new query context for the deletes. we use the same context
// for both, because deleteRows is supposed to handle this.
qcx = holder.MustIndexQueryContext(t, "i")
changed, err := holder.deleteRows(ctx, qcx, row, idx, 0)
if !changed || err != nil {
t.Fatalf("failed to delete row: %v", err)
}
changed, err = holder.deleteRows(ctx, qcx, row, idx, 0)
if err != nil {
t.Fatalf("deleting rows: %v", err)
}
if changed {
t.Fatalf("expected delete to not clear bit but it did")
}
}