featurebase/executor_internal_test.go

// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa

import (
	"context"
	"encoding/json"
	"fmt"
	"reflect"
	"sort"
	"strconv"
	"testing"
	"time"

	"github.com/featurebasedb/featurebase/v3/pql"
	"github.com/featurebasedb/featurebase/v3/testhook"
	"github.com/stretchr/testify/assert"
)

// AssertEqual checks a given RowIdentifiers against expected values.
func (r *RowIdentifiers) AssertEqual(tb testing.TB, other *RowIdentifiers) {
	sort.Slice(r.Rows, func(i, j int) bool { return r.Rows[i] < r.Rows[j] })
	sort.Slice(other.Rows, func(i, j int) bool { return other.Rows[i] < other.Rows[j] })
	if len(r.Rows) != len(other.Rows) {
		tb.Fatalf("row ID mismatch: got %d, expected %d", r.Rows, other.Rows)
	}
	for i := range r.Rows {
		if r.Rows[i] != other.Rows[i] {
			tb.Fatalf("row ID mismatch: got %d, expected %d", r.Rows, other.Rows)
		}
	}
	sort.Strings(r.Keys)
	sort.Strings(other.Keys)
	if len(r.Keys) != len(other.Keys) {
		tb.Fatalf("row keys mismatch: got %s, expected %s", r.Keys, other.Keys)
	}
	for i := range r.Keys {
		if r.Keys[i] != other.Keys[i] {
			tb.Fatalf("row keys mismatch: got %s, expected %s", r.Keys, other.Keys)
		}
	}
}

func TestExecutor_TranslateRowsOnBool(t *testing.T) {
	path, _ := testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
	holder := NewHolder(path, mustHolderConfig())
	defer holder.Close()

	e := &executor{
		Holder:  holder,
		Cluster: NewTestCluster(t, 1),
	}
	if err := e.Holder.Open(); err != nil {
		t.Fatalf("opening holder: %v", err)
	}

	idx, err := e.Holder.CreateIndex("i", IndexOptions{})
	if err != nil {
		t.Fatalf("creating index: %v", err)
	}

	qcx := idx.Txf().NewWritableQcx()
	defer qcx.Abort()

	fb, errb := idx.CreateField("b", OptFieldTypeBool())
	_, errbk := idx.CreateField("bk", OptFieldTypeBool(), OptFieldKeys())
	if errb != nil || errbk != nil {
		t.Fatalf("creating fields %v, %v", errb, errbk)
	}

	_, err1 := fb.SetBit(qcx, 1, 1, nil)
	_, err2 := fb.SetBit(qcx, 2, 2, nil)
	_, err3 := fb.SetBit(qcx, 3, 3, nil)
	if err1 != nil || err2 != nil || err3 != nil {
		t.Fatalf("setting bit %v, %v, %v", err1, err2, err3)
	}

	tests := []struct {
		pql string
	}{
		{pql: "Rows(b)"},
		{pql: "GroupBy(Rows(b))"},
		{pql: "Set(4, b=true)"},
	}

	for _, test := range tests {
		t.Run(test.pql, func(t *testing.T) {
			query, err := pql.ParseString(test.pql)
			if err != nil {
				t.Fatalf("parsing query: %v", err)
			}

			c := query.Calls[0]
			colTranslations, rowTranslations, err := e.preTranslate(context.Background(), "i", c)
			if err != nil {
				t.Fatalf("pre-translating call: %v", err)
			}
			_, err = e.translateCall(c, "i", colTranslations, rowTranslations)
			if err != nil {
				t.Fatalf("translating call: %v", err)
			}
		})
	}
}

func TestFieldRowMarshalJSON(t *testing.T) {
	fr := FieldRow{
		Field:  "blah",
		RowID:  0,
		RowKey: "ha",
	}
	b, err := json.Marshal(fr)
	if err != nil {
		t.Fatalf("marshalling fieldrow: %v", err)
	}
	if string(b) != `{"field":"blah","rowKey":"ha"}` {
		t.Fatalf("unexpected json: %s", b)
	}

	fr = FieldRow{
		Field:  "blah",
		RowID:  2,
		RowKey: "",
	}
	b, err = json.Marshal(fr)
	if err != nil {
		t.Fatalf("marshalling fieldrow: %v", err)
	}
	if string(b) != `{"field":"blah","rowID":2}` {
		t.Fatalf("unexpected json: %s", b)
	}
}

func TestExecutor_GroupCountCondition(t *testing.T) {
	t.Run("satisfiesCondition", func(t *testing.T) {
		type condCheck struct {
			cond string
			exp  bool
		}
		tests := []struct {
			groupCount GroupCount
			checks     []condCheck
		}{
			{
				groupCount: GroupCount{Count: 100},
				checks: []condCheck{
					{cond: "count == 99", exp: false},
					{cond: "count != 99", exp: true},
					{cond: "count < 99", exp: false},
					{cond: "count <= 99", exp: false},
					{cond: "count > 99", exp: true},
					{cond: "count >= 99", exp: true},

					{cond: "count == 100", exp: true},
					{cond: "count != 100", exp: false},
					{cond: "count < 100", exp: false},
					{cond: "count <= 100", exp: true},
					{cond: "count > 100", exp: false},
					{cond: "count >= 100", exp: true},

					{cond: "count == 101", exp: false},
					{cond: "count != 101", exp: true},
					{cond: "count < 101", exp: true},
					{cond: "count <= 101", exp: true},
					{cond: "count > 101", exp: false},
					{cond: "count >= 101", exp: false},

					{cond: "98 < count < 100", exp: false},
					{cond: "98 < count <= 100", exp: true},
					{cond: "98 < count < 101", exp: true},
					{cond: "100 <= count < 102", exp: true},
					{cond: "100 < count < 102", exp: false},
					{cond: "98 <= count <= 102", exp: true},
				},
			},
			{
				groupCount: GroupCount{Agg: 100},
				checks: []condCheck{
					{cond: "sum == 99", exp: false},
					{cond: "sum != 99", exp: true},
					{cond: "sum < 99", exp: false},
					{cond: "sum <= 99", exp: false},
					{cond: "sum > 99", exp: true},
					{cond: "sum >= 99", exp: true},

					{cond: "sum == 100", exp: true},
					{cond: "sum != 100", exp: false},
					{cond: "sum < 100", exp: false},
					{cond: "sum <= 100", exp: true},
					{cond: "sum > 100", exp: false},
					{cond: "sum >= 100", exp: true},

					{cond: "sum == 101", exp: false},
					{cond: "sum != 101", exp: true},
					{cond: "sum < 101", exp: true},
					{cond: "sum <= 101", exp: true},
					{cond: "sum > 101", exp: false},
					{cond: "sum >= 101", exp: false},

					{cond: "98 < sum < 100", exp: false},
					{cond: "98 < sum <= 100", exp: true},
					{cond: "98 < sum < 101", exp: true},
					{cond: "100 <= sum < 102", exp: true},
					{cond: "100 < sum < 102", exp: false},
					{cond: "98 <= sum <= 102", exp: true},
				},
			},
			{
				groupCount: GroupCount{Agg: -100},
				checks: []condCheck{
					{cond: "sum == -99", exp: false},
					{cond: "sum != -99", exp: true},
					{cond: "sum < -99", exp: true},
					{cond: "sum <= -99", exp: true},
					{cond: "sum > -99", exp: false},
					{cond: "sum >= -99", exp: false},

					{cond: "sum == -100", exp: true},
					{cond: "sum != -100", exp: false},
					{cond: "sum < -100", exp: false},
					{cond: "sum <= -100", exp: true},
					{cond: "sum > -100", exp: false},
					{cond: "sum >= -100", exp: true},

					{cond: "sum == -101", exp: false},
					{cond: "sum != -101", exp: true},
					{cond: "sum < -101", exp: false},
					{cond: "sum <= -101", exp: false},
					{cond: "sum > -101", exp: true},
					{cond: "sum >= -101", exp: true},

					{cond: "-100 < sum < -98", exp: false},
					{cond: "-100 <= sum < -98", exp: true},
					{cond: "-101 < sum < -98", exp: true},
					{cond: "-102 < sum <= -100", exp: true},
					{cond: "-102 < sum < -100", exp: false},
					{cond: "-102 <= sum <= -98", exp: true},
				},
			},
		}
		for i, test := range tests {
			t.Run(fmt.Sprintf("test (#%d):", i), func(t *testing.T) {
				for j, check := range test.checks {
					t.Run(fmt.Sprintf("check (#%d):", j), func(t *testing.T) {

						query, err := pql.ParseString(fmt.Sprintf("GroupBy(Rows(a), having=Condition(%s))", check.cond))
						if err != nil {
							t.Fatalf("parsing query: %v", err)
						}
						c := query.Calls[0]
						having := c.Args["having"].(*pql.Call)

						var got bool
						for subj, cond := range having.Args {
							switch subj {
							case "count", "sum":
								condition, ok := cond.(*pql.Condition)
								if !ok {
									t.Fatalf("not a valid condition")
								}
								got = test.groupCount.satisfiesCondition(subj, condition)
							}
						}

						if got != check.exp {
							t.Fatalf("expected: %v, but got: %v", check.exp, got)
						}
					})
				}
			})
		}
	})
}

func TestValCountComparisons(t *testing.T) {
	tests := []struct {
		name       string
		vc         ValCount
		other      ValCount
		expLarger  ValCount
		expSmaller ValCount
	}{
		{
			name: "zero",
		},
		{
			name:       "ints",
			vc:         ValCount{Val: 10, Count: 1},
			other:      ValCount{Val: 3, Count: 2},
			expLarger:  ValCount{Val: 10, Count: 1},
			expSmaller: ValCount{Val: 3, Count: 2},
		},
		{
			name:       "floats",
			vc:         ValCount{FloatVal: 10.2, Count: 1},
			other:      ValCount{FloatVal: 3.4, Count: 2},
			expLarger:  ValCount{FloatVal: 10.2, Count: 1},
			expSmaller: ValCount{FloatVal: 3.4, Count: 2},
		},
		{
			name:       "intsEquality",
			vc:         ValCount{Val: 10, Count: 1},
			other:      ValCount{Val: 10, Count: 2},
			expLarger:  ValCount{Val: 10, Count: 3},
			expSmaller: ValCount{Val: 10, Count: 3},
		},
		{
			name:       "floatsEquality",
			vc:         ValCount{FloatVal: 10.7, Count: 1},
			other:      ValCount{FloatVal: 10.7, Count: 2},
			expLarger:  ValCount{FloatVal: 10.7, Count: 3},
			expSmaller: ValCount{FloatVal: 10.7, Count: 3},
		},
		{
			name:       "timestampEquality",
			vc:         ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
			other:      ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
			expLarger:  ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 2},
			expSmaller: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 2},
		},
		{
			name:       "timestamp",
			vc:         ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
			other:      ValCount{Val: 1587399600, TimestampVal: time.Unix(0, 1587399600*int64(time.Second)), Count: 1},
			expLarger:  ValCount{Val: 1587399600, TimestampVal: time.Unix(0, 1587399600*int64(time.Second)), Count: 1},
			expSmaller: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1},
		},
	}

	for i, test := range tests {
		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
			gotLarger := test.vc.larger(test.other)
			if gotLarger != test.expLarger {
				t.Fatalf("larger failed, expected:\n%+v\ngot:\n%+v", test.expLarger, gotLarger)
			}

			gotSmaller := test.vc.smaller(test.other)
			if gotSmaller != test.expSmaller {
				t.Fatalf("smaller failed, expected:\n%+v\ngot:\n%+v", test.expSmaller, gotSmaller)
			}
		})
	}
}

func TestToNegInt64(t *testing.T) {
	tests := []struct {
		u64      uint64
		i64      int64
		overflow bool
	}{
		{
			u64: uint64(1 << 63),
			i64: int64(-1 << 63),
		},
		{
			u64: uint64(1<<63) - 1,
			i64: int64(-1<<63) + 1,
		},
		{
			u64:      uint64(1<<63) + 1,
			overflow: true,
		},
	}

	for _, tc := range tests {
		val, err := toNegInt64(tc.u64)
		if err != nil && !tc.overflow {
			t.Fatalf("error: %+v, expected: %+v", err, tc)
		}

		if val != tc.i64 {
			t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
		}
	}
}

func TestToInt64(t *testing.T) {
	tests := []struct {
		u64      uint64
		i64      int64
		overflow bool
	}{
		{
			u64: uint64(1<<63) - 1,
			i64: 1<<63 - 1,
		},
		{
			u64: uint64(0),
			i64: 0,
		},
		{
			u64:      uint64(1 << 63),
			overflow: true,
		},
		{
			u64:      1<<64 - 1,
			overflow: true,
		},
	}

	for _, tc := range tests {
		val, err := toInt64(tc.u64)
		if err != nil && !tc.overflow {
			t.Fatalf("error: %+v, expected: %+v", err, tc)
		}

		if val != tc.i64 {
			t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
		}
	}
}

func TestGetSorter(t *testing.T) {
	tests := []struct {
		sortSpec string
		expGCS   *groupCountSorter
		expErr   string
	}{
		{
			sortSpec: "count asc",
			expGCS:   &groupCountSorter{fields: []int{-1}, order: []order{asc}},
		},
		{
			sortSpec: "count    asc",
			expGCS:   &groupCountSorter{fields: []int{-1}, order: []order{asc}},
		},
		{
			sortSpec: "  count asc",
			expGCS:   &groupCountSorter{fields: []int{-1}, order: []order{asc}},
		},
		{
			sortSpec: "  count asc    ",
			expGCS:   &groupCountSorter{fields: []int{-1}, order: []order{asc}},
		},
		{
			sortSpec: "count",
			expGCS:   &groupCountSorter{fields: []int{-1}, order: []order{desc}},
		},
		{
			sortSpec: "sum asc",
			expGCS:   &groupCountSorter{fields: []int{-2}, order: []order{asc}},
		},
		{
			sortSpec: "aggregate asc",
			expGCS:   &groupCountSorter{fields: []int{-2}, order: []order{asc}},
		},
		{
			sortSpec: "boondoggle asc",
			expErr:   "sorting is only supported on count, aggregate, or sum, not 'boondoggle'",
		},
		{
			sortSpec: "sum asc, count desc",
			expGCS:   &groupCountSorter{fields: []int{-2, -1}, order: []order{asc, desc}},
		},
		{
			sortSpec: "count asc, sum desc",
			expGCS:   &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}},
		},
		{
			sortSpec: " count  asc ,  sum  desc ",
			expGCS:   &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}},
		},
		{
			sortSpec: " count  asc ,  sum  desc blah",
			expErr:   "parsing sort directive: 'sum  desc blah': too many elements",
		},
		{
			sortSpec: "count asc, sum fesc",
			expErr:   "unknown sort direction 'fesc'",
		},
		{
			sortSpec: " , sum fesc",
			expErr:   "invalid sorting directive: ''",
		},
		{
			// weird and useless, but I guess fine?
			sortSpec: "count  asc,count asc ",
			expGCS:   &groupCountSorter{fields: []int{-1, -1}, order: []order{asc, asc}},
		},
	}

	for i, tst := range tests {
		t.Run(fmt.Sprintf("%s_%d", tst.sortSpec, i), func(t *testing.T) {
			gcs, err := getSorter(tst.sortSpec)
			if err != nil {
				if tst.expErr == "" {
					t.Errorf("unexpected error: %v", err)
					return
				}
				if tst.expErr != err.Error() {
					t.Errorf("mismatched errors got: '%v', exp: '%s'", err, tst.expErr)
				}
				return
			}
			if !reflect.DeepEqual(gcs, tst.expGCS) {
				t.Errorf("exp:\n%+v\ngot:\n%v\n", tst.expGCS, gcs)
			}

		})
	}
}

func TestExecutorSafeCopyDistinctTimestamp(t *testing.T) {
	result := DistinctTimestamp{Values: []string{"test", "test"}, Name: "test"}
	results := make([]interface{}, 1)
	results[0] = result

	response := QueryResponse{Results: results, Err: nil, Profile: nil}
	copied := safeCopy(response)
	if !reflect.DeepEqual(copied.Results, response.Results) {
		t.Fatalf("Did not copy results. got %+v, want %+v", copied.Results, response.Results)
	}
}

func TestGetScaledInt(t *testing.T) {
	f := OpenField(t, OptFieldTypeTimestamp(time.Now(), "ms"))
	// check that fields with type timestamp return the int64 passed in to getScaledInt with nil err
	v := time.Now().Unix()
	res, err := getScaledInt(f.Field, v)
	if err != nil {
		t.Errorf("got error %v, expected nil", err)
	}
	if !reflect.DeepEqual(res, v) {
		t.Errorf("expected %v, got %v", v, res)
	}

}

func TestDistinctTimestampUnion(t *testing.T) {
	cases := []struct {
		name     string
		a        DistinctTimestamp
		b        DistinctTimestamp
		expected DistinctTimestamp
	}{
		{
			name:     "empty other",
			a:        DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
			b:        DistinctTimestamp{Name: "a", Values: []string{}},
			expected: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
		},
		{
			name:     "one more in other",
			a:        DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
			b:        DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c", "d"}},
			expected: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c", "d"}},
		},
	}
	for _, test := range cases {
		t.Run(test.name, func(t *testing.T) {
			res := test.a.Union(test.b)
			allThere := true
			for _, val := range res.Values {
				here := false
				for _, expected := range test.expected.Values {
					if val == expected {
						here = true
						break
					}
				}
				allThere = allThere && here
			}
			if !allThere {
				t.Errorf("expected %v, got %v", test.expected, res)
			}
		})
	}
}

func TestExecutor_DeleteRows(t *testing.T) {
	path, _ := testhook.TempDir(t, "pilosa-executor-")
	holder := NewHolder(path, mustHolderConfig())
	defer holder.Close()

	if err := holder.Open(); err != nil {
		t.Fatalf("opening holder: %v", err)
	}

	idx, err := holder.CreateIndex("i", IndexOptions{TrackExistence: true})
	if err != nil {
		t.Fatalf("creating index: %v", err)
	}

	f, err := idx.CreateField("f", OptFieldTypeDefault())
	if err != nil {
		t.Fatalf("creating field: %v", err)
	}

	qcx := idx.Txf().NewWritableQcx()
	defer qcx.Abort()
	if _, err = f.SetBit(qcx, 1, 1, nil); err != nil {
		t.Fatalf("setting bit: %v", err)
	}

	// We rely here on the fact that write Qcx autocommit constantly.
	row, err := f.Row(qcx, 1)
	if err != nil {
		t.Fatalf("failed to read row: %v", err)
	}

	ctx := context.Background()
	changed, err := DeleteRows(ctx, row, idx, 0)
	if !changed || err != nil {
		t.Fatalf("failed to delete row: %v", err)
	}

	changed, err = DeleteRows(ctx, row, idx, 0)
	if changed {
		t.Fatalf("expected delete to not clear bit but it did")
	}
}