scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
featurebase/executor_internal_test.go
Seebs cf94181c89 significant refactor of test setup and teardown 
We centralize the creation paths for test indexes, fields,
etcetera so they all have a common path, all using standard
test holders. There's still two versions, one for test.* functions
and one for internal. They do share a TestHolderConfig though.

Large hunks of the related APIs are simplified/streamlined.
* Fragments are always created with a Field and don't need
  a workaround in case they don't have it.
* Creation of test fragments, etc., use optional FieldOptions
  but don't specify names because they're all using new holders
  for each thing created anyway. This dramatically reduces
  the complexity of the calls.
* test fragments are created inside test views which are created
  inside test fields, etcetera, so everything is using the same
  logic; test views aren't bypassing the other layers, they're
  creating themselves normally within a field.
* Quite a few things now use the standard runtime/production
  logic instead of being custom workarounds; for instance, instead
  of `mustOpenMutexFragment` creating a fragment and then creating
  a mutex vector for it, we just create a mutex-typed field and
  have the normal runtime code do this.
* Similarly, we now use the same field creation logic that production
  does, instead of having our own test-only thing that validates
  field names directly, so our test that we're validating field names
  is actually testing the runtime code. Yay.
* fragSpec goes away. it was a replacement for fragProxy which existed
  to solve memory allocation problems but replaced them with interface
  overhead problems. Now we just have pointers to things and maintain
  valid data structures.
* Many panics are now Fatal or Fatalf calls.
* Some specific bugs fixed, like a cluster which was requested and
  then had its first node directly overwritten, which isn't valid with
  shared clusters.
* Drop the temp-dir test flag and TempDir variable, we can just use
  $TMPDIR.
* Drop a benchmark of "write file to disk" that was purely a benchmark
  of file write speed, not a benchmark of rendering the data that needs
  to be written.
* Drop the unused "flags" parameter to fragment creation, which was
  only used back when we changed the BSI format.
* Use holder.Txf() rather than index.Txf(). The TxFactory has to be
  holder-level anyway, referring to it via the index is misleading.
* Test holders automatically close themselves and delete themselves,
  we remove various other things that thought they were responsible
  for deleting themselves.
2022-09-23 16:56:27 -05:00

597 lines
16 KiB
Go
Raw Blame History

// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"context"
"encoding/json"
"fmt"
"reflect"
"sort"
"strconv"
"testing"
"time"
"github.com/molecula/featurebase/v3/pql"
)
// AssertEqual checks a given RowIdentifiers against expected values.
func (r *RowIdentifiers) AssertEqual(tb testing.TB, other *RowIdentifiers) {
sort.Slice(r.Rows, func(i, j int) bool { return r.Rows[i] < r.Rows[j] })
sort.Slice(other.Rows, func(i, j int) bool { return other.Rows[i] < other.Rows[j] })
if len(r.Rows) != len(other.Rows) {
tb.Fatalf("row ID mismatch: got %d, expected %d", r.Rows, other.Rows)
}
for i := range r.Rows {
if r.Rows[i] != other.Rows[i] {
tb.Fatalf("row ID mismatch: got %d, expected %d", r.Rows, other.Rows)
}
}
sort.Strings(r.Keys)
sort.Strings(other.Keys)
if len(r.Keys) != len(other.Keys) {
tb.Fatalf("row keys mismatch: got %s, expected %s", r.Keys, other.Keys)
}
for i := range r.Keys {
if r.Keys[i] != other.Keys[i] {
tb.Fatalf("row keys mismatch: got %s, expected %s", r.Keys, other.Keys)
}
}
}
func TestExecutor_TranslateRowsOnBool(t *testing.T) {
holder := newTestHolder(t)
e := &executor{
Holder: holder,
Cluster: NewTestCluster(t, 1),
}
idx, err := e.Holder.CreateIndex("i", IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
qcx := holder.Txf().NewWritableQcx()
defer qcx.Abort()
fb, errb := idx.CreateField("b", OptFieldTypeBool())
_, errbk := idx.CreateField("bk", OptFieldTypeBool(), OptFieldKeys())
if errb != nil || errbk != nil {
t.Fatalf("creating fields %v, %v", errb, errbk)
}
_, err1 := fb.SetBit(qcx, 1, 1, nil)
_, err2 := fb.SetBit(qcx, 2, 2, nil)
_, err3 := fb.SetBit(qcx, 3, 3, nil)
if err1 != nil || err2 != nil || err3 != nil {
t.Fatalf("setting bit %v, %v, %v", err1, err2, err3)
}
tests := []struct {
pql string
}{
{pql: "Rows(b)"},
{pql: "GroupBy(Rows(b))"},
{pql: "Set(4, b=true)"},
}
for _, test := range tests {
t.Run(test.pql, func(t *testing.T) {
query, err := pql.ParseString(test.pql)
if err != nil {
t.Fatalf("parsing query: %v", err)
}
c := query.Calls[0]
colTranslations, rowTranslations, err := e.preTranslate(context.Background(), "i", c)
if err != nil {
t.Fatalf("pre-translating call: %v", err)
}
_, err = e.translateCall(c, "i", colTranslations, rowTranslations)
if err != nil {
t.Fatalf("translating call: %v", err)
}
})
}
}
func TestFieldRowMarshalJSON(t *testing.T) {
fr := FieldRow{
Field: "blah",
RowID: 0,
RowKey: "ha",
}
b, err := json.Marshal(fr)
if err != nil {
t.Fatalf("marshalling fieldrow: %v", err)
}
if string(b) != `{"field":"blah","rowKey":"ha"}` {
t.Fatalf("unexpected json: %s", b)
}
fr = FieldRow{
Field: "blah",
RowID: 2,
RowKey: "",
}
b, err = json.Marshal(fr)
if err != nil {
t.Fatalf("marshalling fieldrow: %v", err)
}
if string(b) != `{"field":"blah","rowID":2}` {
t.Fatalf("unexpected json: %s", b)
}
}
func TestExecutor_GroupCountCondition(t *testing.T) {
t.Run("satisfiesCondition", func(t *testing.T) {
type condCheck struct {
cond string
exp bool
}
tests := []struct {
groupCount GroupCount
checks []condCheck
}{
{
groupCount: GroupCount{Count: 100},
checks: []condCheck{
{cond: "count == 99", exp: false},
{cond: "count != 99", exp: true},
{cond: "count < 99", exp: false},
{cond: "count <= 99", exp: false},
{cond: "count > 99", exp: true},
{cond: "count >= 99", exp: true},
{cond: "count == 100", exp: true},
{cond: "count != 100", exp: false},
{cond: "count < 100", exp: false},
{cond: "count <= 100", exp: true},
{cond: "count > 100", exp: false},
{cond: "count >= 100", exp: true},
{cond: "count == 101", exp: false},
{cond: "count != 101", exp: true},
{cond: "count < 101", exp: true},
{cond: "count <= 101", exp: true},
{cond: "count > 101", exp: false},
{cond: "count >= 101", exp: false},
{cond: "98 < count < 100", exp: false},
{cond: "98 < count <= 100", exp: true},
{cond: "98 < count < 101", exp: true},
{cond: "100 <= count < 102", exp: true},
{cond: "100 < count < 102", exp: false},
{cond: "98 <= count <= 102", exp: true},
},
},
{
groupCount: GroupCount{Agg: 100},
checks: []condCheck{
{cond: "sum == 99", exp: false},
{cond: "sum != 99", exp: true},
{cond: "sum < 99", exp: false},
{cond: "sum <= 99", exp: false},
{cond: "sum > 99", exp: true},
{cond: "sum >= 99", exp: true},
{cond: "sum == 100", exp: true},
{cond: "sum != 100", exp: false},
{cond: "sum < 100", exp: false},
{cond: "sum <= 100", exp: true},
{cond: "sum > 100", exp: false},
{cond: "sum >= 100", exp: true},
{cond: "sum == 101", exp: false},
{cond: "sum != 101", exp: true},
{cond: "sum < 101", exp: true},
{cond: "sum <= 101", exp: true},
{cond: "sum > 101", exp: false},
{cond: "sum >= 101", exp: false},
{cond: "98 < sum < 100", exp: false},
{cond: "98 < sum <= 100", exp: true},
{cond: "98 < sum < 101", exp: true},
{cond: "100 <= sum < 102", exp: true},
{cond: "100 < sum < 102", exp: false},
{cond: "98 <= sum <= 102", exp: true},
},
},
{
groupCount: GroupCount{Agg: -100},
checks: []condCheck{
{cond: "sum == -99", exp: false},
{cond: "sum != -99", exp: true},
{cond: "sum < -99", exp: true},
{cond: "sum <= -99", exp: true},
{cond: "sum > -99", exp: false},
{cond: "sum >= -99", exp: false},
{cond: "sum == -100", exp: true},
{cond: "sum != -100", exp: false},
{cond: "sum < -100", exp: false},
{cond: "sum <= -100", exp: true},
{cond: "sum > -100", exp: false},
{cond: "sum >= -100", exp: true},
{cond: "sum == -101", exp: false},
{cond: "sum != -101", exp: true},
{cond: "sum < -101", exp: false},
{cond: "sum <= -101", exp: false},
{cond: "sum > -101", exp: true},
{cond: "sum >= -101", exp: true},
{cond: "-100 < sum < -98", exp: false},
{cond: "-100 <= sum < -98", exp: true},
{cond: "-101 < sum < -98", exp: true},
{cond: "-102 < sum <= -100", exp: true},
{cond: "-102 < sum < -100", exp: false},
{cond: "-102 <= sum <= -98", exp: true},
},
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("test (#%d):", i), func(t *testing.T) {
for j, check := range test.checks {
t.Run(fmt.Sprintf("check (#%d):", j), func(t *testing.T) {
query, err := pql.ParseString(fmt.Sprintf("GroupBy(Rows(a), having=Condition(%s))", check.cond))
if err != nil {
t.Fatalf("parsing query: %v", err)
}
c := query.Calls[0]
having := c.Args["having"].(*pql.Call)
var got bool
for subj, cond := range having.Args {
switch subj {
case "count", "sum":
condition, ok := cond.(*pql.Condition)
if !ok {
t.Fatalf("not a valid condition")
}
got = test.groupCount.satisfiesCondition(subj, condition)
}
}
if got != check.exp {
t.Fatalf("expected: %v, but got: %v", check.exp, got)
}
})
}
})
}
})
}
func TestValCountComparisons(t *testing.T) {
tests := []struct {
name string
vc ValCount
other ValCount
expLarger ValCount
expSmaller ValCount
}{
{
name: "zero",
},
{
name: "ints",
vc: ValCount{Val: 10, Count: 1},
other: ValCount{Val: 3, Count: 2},
expLarger: ValCount{Val: 10, Count: 1},
expSmaller: ValCount{Val: 3, Count: 2},
},
{
name: "floats",
vc: ValCount{FloatVal: 10.2, Count: 1},
other: ValCount{FloatVal: 3.4, Count: 2},
expLarger: ValCount{FloatVal: 10.2, Count: 1},
expSmaller: ValCount{FloatVal: 3.4, Count: 2},
},
{
name: "intsEquality",
vc: ValCount{Val: 10, Count: 1},
other: ValCount{Val: 10, Count: 2},
expLarger: ValCount{Val: 10, Count: 3},
expSmaller: ValCount{Val: 10, Count: 3},
},
{
name: "floatsEquality",
vc: ValCount{FloatVal: 10.7, Count: 1},
other: ValCount{FloatVal: 10.7, Count: 2},
expLarger: ValCount{FloatVal: 10.7, Count: 3},
expSmaller: ValCount{FloatVal: 10.7, Count: 3},
},
{
name: "timestampEquality",
vc: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
other: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
expLarger: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 2},
expSmaller: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 2},
},
{
name: "timestamp",
vc: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
other: ValCount{Val: 1587399600, TimestampVal: time.Unix(0, 1587399600*int64(time.Second)), Count: 1},
expLarger: ValCount{Val: 1587399600, TimestampVal: time.Unix(0, 1587399600*int64(time.Second)), Count: 1},
expSmaller: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
},
}
for i, test := range tests {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
gotLarger := test.vc.larger(test.other)
if gotLarger != test.expLarger {
t.Fatalf("larger failed, expected:\n%+v\ngot:\n%+v", test.expLarger, gotLarger)
}
gotSmaller := test.vc.smaller(test.other)
if gotSmaller != test.expSmaller {
t.Fatalf("smaller failed, expected:\n%+v\ngot:\n%+v", test.expSmaller, gotSmaller)
}
})
}
}
func TestToNegInt64(t *testing.T) {
tests := []struct {
u64 uint64
i64 int64
overflow bool
}{
{
u64: uint64(1 << 63),
i64: int64(-1 << 63),
},
{
u64: uint64(1<<63) - 1,
i64: int64(-1<<63) + 1,
},
{
u64: uint64(1<<63) + 1,
overflow: true,
},
}
for _, tc := range tests {
val, err := toNegInt64(tc.u64)
if err != nil && !tc.overflow {
t.Fatalf("error: %+v, expected: %+v", err, tc)
}
if val != tc.i64 {
t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
}
}
}
func TestToInt64(t *testing.T) {
tests := []struct {
u64 uint64
i64 int64
overflow bool
}{
{
u64: uint64(1<<63) - 1,
i64: 1<<63 - 1,
},
{
u64: uint64(0),
i64: 0,
},
{
u64: uint64(1 << 63),
overflow: true,
},
{
u64: 1<<64 - 1,
overflow: true,
},
}
for _, tc := range tests {
val, err := toInt64(tc.u64)
if err != nil && !tc.overflow {
t.Fatalf("error: %+v, expected: %+v", err, tc)
}
if val != tc.i64 {
t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
}
}
}
func TestGetSorter(t *testing.T) {
tests := []struct {
sortSpec string
expGCS *groupCountSorter
expErr string
}{
{
sortSpec: "count asc",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: "count asc",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: " count asc",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: " count asc ",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: "count",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{desc}},
},
{
sortSpec: "sum asc",
expGCS: &groupCountSorter{fields: []int{-2}, order: []order{asc}},
},
{
sortSpec: "aggregate asc",
expGCS: &groupCountSorter{fields: []int{-2}, order: []order{asc}},
},
{
sortSpec: "boondoggle asc",
expErr: "sorting is only supported on count, aggregate, or sum, not 'boondoggle'",
},
{
sortSpec: "sum asc, count desc",
expGCS: &groupCountSorter{fields: []int{-2, -1}, order: []order{asc, desc}},
},
{
sortSpec: "count asc, sum desc",
expGCS: &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}},
},
{
sortSpec: " count asc , sum desc ",
expGCS: &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}},
},
{
sortSpec: " count asc , sum desc blah",
expErr: "parsing sort directive: 'sum desc blah': too many elements",
},
{
sortSpec: "count asc, sum fesc",
expErr: "unknown sort direction 'fesc'",
},
{
sortSpec: " , sum fesc",
expErr: "invalid sorting directive: ''",
},
{
// weird and useless, but I guess fine?
sortSpec: "count asc,count asc ",
expGCS: &groupCountSorter{fields: []int{-1, -1}, order: []order{asc, asc}},
},
}
for i, tst := range tests {
t.Run(fmt.Sprintf("%s_%d", tst.sortSpec, i), func(t *testing.T) {
gcs, err := getSorter(tst.sortSpec)
if err != nil {
if tst.expErr == "" {
t.Errorf("unexpected error: %v", err)
return
}
if tst.expErr != err.Error() {
t.Errorf("mismatched errors got: '%v', exp: '%s'", err, tst.expErr)
}
return
}
if !reflect.DeepEqual(gcs, tst.expGCS) {
t.Errorf("exp:\n%+v\ngot:\n%v\n", tst.expGCS, gcs)
}
})
}
}
func TestExecutorSafeCopyDistinctTimestamp(t *testing.T) {
result := DistinctTimestamp{Values: []string{"test", "test"}, Name: "test"}
results := make([]interface{}, 1)
results[0] = result
response := QueryResponse{Results: results, Err: nil, Profile: nil}
copied := safeCopy(response)
if !reflect.DeepEqual(copied.Results, response.Results) {
t.Fatalf("Did not copy results. got %+v, want %+v", copied.Results, response.Results)
}
}
func TestGetScaledInt(t *testing.T) {
_, _, f := newTestField(t, OptFieldTypeTimestamp(time.Now(), "ms"))
// check that fields with type timestamp return the int64 passed in to getScaledInt with nil err
v := time.Now().Unix()
res, err := getScaledInt(f, v)
if err != nil {
t.Errorf("got error %v, expected nil", err)
}
if !reflect.DeepEqual(res, v) {
t.Errorf("expected %v, got %v", v, res)
}
}
func TestDistinctTimestampUnion(t *testing.T) {
cases := []struct {
name string
a DistinctTimestamp
b DistinctTimestamp
expected DistinctTimestamp
}{
{
name: "empty other",
a: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
b: DistinctTimestamp{Name: "a", Values: []string{}},
expected: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
},
{
name: "one more in other",
a: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
b: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c", "d"}},
expected: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c", "d"}},
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
res := test.a.Union(test.b)
allThere := true
for _, val := range res.Values {
here := false
for _, expected := range test.expected.Values {
if val == expected {
here = true
break
}
}
allThere = allThere && here
}
if !allThere {
t.Errorf("expected %v, got %v", test.expected, res)
}
})
}
}
func TestExecutor_DeleteRows(t *testing.T) {
holder := newTestHolder(t)
idx, err := holder.CreateIndex("i", IndexOptions{TrackExistence: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
f, err := idx.CreateField("f")
if err != nil {
t.Fatalf("creating field: %v", err)
}
qcx := holder.Txf().NewWritableQcx()
defer qcx.Abort()
if _, err = f.SetBit(qcx, 1, 1, nil); err != nil {
t.Fatalf("setting bit: %v", err)
}
// We rely here on the fact that write Qcx autocommit constantly.
row, err := f.Row(qcx, 1)
if err != nil {
t.Fatalf("failed to read row: %v", err)
}
ctx := context.Background()
changed, err := DeleteRows(ctx, row, idx, 0)
if !changed || err != nil {
t.Fatalf("failed to delete row: %v", err)
}
changed, err = DeleteRows(ctx, row, idx, 0)
if changed {
t.Fatalf("expected delete to not clear bit but it did")
}
}
Reference in a new issue View git blame Copy permalink