mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 02:44:59 +00:00
dad8211709
For arbitrary mod values m, greater than zero, (x%m + 1) != 0 is always true What we almost certainly meant was x%(m+1) == 0 which would give you all the bits in row 0, half the bits in row 1, etcetera. Also, we drop to doing a quarter-shard because why not.
281 lines
10 KiB
Go
281 lines
10 KiB
Go
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
package pilosa_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"testing"
|
|
"time"
|
|
|
|
pilosa "github.com/featurebasedb/featurebase/v3"
|
|
"github.com/featurebasedb/featurebase/v3/disco"
|
|
"github.com/featurebasedb/featurebase/v3/test"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestExecutor_DeleteRecords(t *testing.T) {
|
|
indexName := "i"
|
|
setup := func(t *testing.T, r *require.Assertions, c *test.Cluster) {
|
|
t.Helper()
|
|
c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, "setfield")
|
|
c.ImportBits(t, indexName, "setfield", [][2]uint64{
|
|
{0, 0},
|
|
{0, 1},
|
|
{0, ShardWidth + 2},
|
|
{10, 2},
|
|
{10, ShardWidth},
|
|
{10, 2 * ShardWidth},
|
|
{10, ShardWidth + 1},
|
|
{20, ShardWidth},
|
|
})
|
|
c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, "bsi", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
|
|
c.ImportIntID(t, indexName, "bsi", []test.IntID{
|
|
{ID: 0, Val: 4},
|
|
{ID: 2, Val: 8},
|
|
})
|
|
c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, "timefield", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
|
|
c.ImportBitsWithTimestamp(t, indexName, "timefield", [][2]uint64{
|
|
{0, 0},
|
|
{0, 1},
|
|
{0, 1},
|
|
{0, 1},
|
|
{0, 1},
|
|
}, []int64{
|
|
time.Date(2020, time.January, 2, 15, 45, 0, 0, time.UTC).Unix(),
|
|
time.Date(2019, time.January, 2, 16, 45, 0, 0, time.UTC).Unix(),
|
|
time.Date(2019, time.January, 2, 16, 45, 0, 0, time.UTC).Unix(),
|
|
time.Date(2019, time.January, 2, 17, 45, 0, 0, time.UTC).Unix(),
|
|
time.Date(2019, time.January, 2, 17, 45, 0, 0, time.UTC).Unix(),
|
|
})
|
|
|
|
}
|
|
setupBig := func(t *testing.T, r *require.Assertions, c *test.Cluster, Rows uint64) {
|
|
t.Helper()
|
|
fieldName := "setfield"
|
|
c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, fieldName)
|
|
// we don't need to populate the whole thing, just enough to get a sample of it
|
|
width := uint64(ShardWidth / 4)
|
|
rows := make([][2]uint64, width*Rows)
|
|
n := 0
|
|
// populate rows with decreasing density
|
|
for columnID := uint64(0); columnID < width; columnID++ {
|
|
for rowID := uint64(0); rowID < Rows; rowID++ {
|
|
if (columnID % (rowID + 1)) == 0 {
|
|
rows[n] = [2]uint64{rowID, columnID}
|
|
n++
|
|
}
|
|
}
|
|
}
|
|
c.ImportBits(t, indexName, "setfield", rows[:n])
|
|
}
|
|
|
|
setupKeys := func(t *testing.T, r *require.Assertions, c *test.Cluster) {
|
|
t.Helper()
|
|
c.CreateField(t, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "timefield", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
|
|
c.ImportTimeQuantumKey(t, indexName, "timefield", []test.TimeQuantumKey{
|
|
{RowKey: "fish", ColKey: "one", Ts: time.Date(2019, time.January, 2, 17, 45, 0, 0, time.UTC).Unix()},
|
|
{RowKey: "fish", ColKey: "one", Ts: time.Date(2020, time.January, 2, 17, 45, 0, 0, time.UTC).Unix()},
|
|
{RowKey: "fish", ColKey: "two", Ts: time.Date(2019, time.January, 3, 17, 45, 0, 0, time.UTC).Unix()},
|
|
})
|
|
c.CreateField(t, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "keystuff")
|
|
c.ImportIDKey(t, indexName, "keystuff", []test.KeyID{
|
|
{ID: 1, Key: "A"},
|
|
{ID: 2, Key: "B"},
|
|
{ID: 3, Key: "C"},
|
|
{ID: 4, Key: "D"},
|
|
})
|
|
|
|
}
|
|
setupOverlap := func(t *testing.T, r *require.Assertions, c *test.Cluster) {
|
|
t.Helper()
|
|
c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, "setfield")
|
|
c.ImportBits(t, indexName, "setfield", [][2]uint64{
|
|
{0, 0},
|
|
{0, 1},
|
|
{1, 1},
|
|
{2, 1},
|
|
{3, 1},
|
|
{0, ShardWidth},
|
|
{2, ShardWidth},
|
|
{4, ShardWidth},
|
|
{6, ShardWidth},
|
|
})
|
|
}
|
|
tearDown := func(t *testing.T, require *require.Assertions, c *test.Cluster) {
|
|
t.Helper()
|
|
api := c.GetPrimary().API
|
|
err := api.DeleteIndex(context.Background(), indexName)
|
|
require.NoErrorf(err, "DeleteIndex %v", indexName)
|
|
}
|
|
require := require.New(t)
|
|
t.Run("DeleteRecords", func(t *testing.T) {
|
|
c := test.MustNewCluster(t, 1)
|
|
for _, n := range c.Nodes {
|
|
n.Config.Cluster.ReplicaN = 1
|
|
}
|
|
err := c.Start()
|
|
require.NoError(err, "Start cluster DeleteRecords")
|
|
defer c.Close()
|
|
|
|
t.Run("Delete", func(t *testing.T) {
|
|
setup(t, require, c)
|
|
defer tearDown(t, require, c)
|
|
resp := c.Query(t, indexName, `Extract(All())`)
|
|
m := resp.Results[0].(pilosa.ExtractedTable)
|
|
before := convert(m.Columns)
|
|
require.Equal([]uint64{0, 1, 2, ShardWidth, ShardWidth + 1, ShardWidth + 2, 2 * ShardWidth}, before, "these records are expected")
|
|
resp = c.Query(t, indexName, fmt.Sprintf(`Delete(ConstRow(columns=[1,2,3,%v]))`, ShardWidth+1))
|
|
require.NotNil(resp, "Response should not be nil")
|
|
require.NotEmpty(resp.Results)
|
|
require.Equal(true, resp.Results[0], "Change should have happened")
|
|
|
|
resp = c.Query(t, indexName, `Extract(All())`)
|
|
|
|
//Note none of the removed records should remain
|
|
m = resp.Results[0].(pilosa.ExtractedTable)
|
|
after := convert(m.Columns)
|
|
require.Equal([]uint64{0, ShardWidth, ShardWidth + 2, 2 * ShardWidth}, after, "these records should be remaining")
|
|
})
|
|
t.Run("DeleteKey", func(t *testing.T) {
|
|
setupKeys(t, require, c)
|
|
defer tearDown(t, require, c)
|
|
resp := c.Query(t, indexName, `Extract(All())`)
|
|
m := resp.Results[0].(pilosa.ExtractedTable)
|
|
before := convertKey(m.Columns)
|
|
require.Equal([]string{"one", "A", "B", "C", "D", "two"}, before, "these keyed records before")
|
|
resp = c.Query(t, indexName, `Delete(ConstRow(columns=["A","one"]))`)
|
|
require.NotEmpty(resp.Results)
|
|
require.Equal(true, resp.Results[0], "Change should have happened")
|
|
|
|
resp = c.Query(t, indexName, `Extract(All())`)
|
|
m = resp.Results[0].(pilosa.ExtractedTable)
|
|
after := convertKey(m.Columns)
|
|
require.Equal([]string{"B", "C", "D", "two"}, after, "these keyed records after delete")
|
|
//validate that column keys got deleted
|
|
node := c.GetNode(0)
|
|
keys := []string{"A", "one"}
|
|
res, err := node.API.FindIndexKeys(context.Background(), indexName, keys...)
|
|
require.Nil(err)
|
|
require.Empty(res)
|
|
})
|
|
t.Run("Delete Row", func(t *testing.T) {
|
|
setup(t, require, c)
|
|
defer tearDown(t, require, c)
|
|
resp := c.Query(t, indexName, `Delete(Row(setfield=20))`)
|
|
require.NotNil(resp, "Response should not be nil")
|
|
require.NotEmpty(resp.Results)
|
|
require.Equal(true, resp.Results[0], "Change should have happened")
|
|
|
|
resp = c.Query(t, indexName, `Extract(All())`)
|
|
|
|
//Note none of the removed records should remain
|
|
m := resp.Results[0].(pilosa.ExtractedTable)
|
|
after := convert(m.Columns)
|
|
require.Equal([]uint64{0, 1, 2, ShardWidth + 1, ShardWidth + 2, 2 * ShardWidth}, after, "these records are expected")
|
|
})
|
|
t.Run("Delete Not Row", func(t *testing.T) {
|
|
setup(t, require, c)
|
|
defer tearDown(t, require, c)
|
|
resp := c.Query(t, indexName, `Delete(Not(Row(setfield=20)))`)
|
|
require.NotNil(resp, "Response should not be nil")
|
|
require.NotEmpty(resp.Results)
|
|
require.Equal(true, resp.Results[0], "Change should have happened")
|
|
|
|
resp = c.Query(t, indexName, `Extract(All())`)
|
|
|
|
//Note none of the removed records should remain
|
|
m := resp.Results[0].(pilosa.ExtractedTable)
|
|
after := convert(m.Columns)
|
|
require.Equal([]uint64{ShardWidth}, after, "these records are expected")
|
|
})
|
|
t.Run("Delete All", func(t *testing.T) {
|
|
setup(t, require, c)
|
|
defer tearDown(t, require, c)
|
|
resp := c.Query(t, indexName, `Count(All())`)
|
|
//Note none of the removed records should remain
|
|
before := resp.Results[0].(uint64)
|
|
require.Equal(uint64(7), before, "these records are expected")
|
|
|
|
resp = c.Query(t, indexName, `Delete(All())`)
|
|
require.NotNil(resp, "Response should not be nil")
|
|
require.NotEmpty(resp.Results)
|
|
require.Equal(true, resp.Results[0], "Change should have happened")
|
|
|
|
resp = c.Query(t, indexName, `Count(All())`)
|
|
//Note none of the removed records should remain
|
|
after := resp.Results[0].(uint64)
|
|
require.Equal(uint64(0), after, "these records are expected")
|
|
})
|
|
t.Run("DeleteOverlap", func(t *testing.T) {
|
|
setupOverlap(t, require, c)
|
|
defer tearDown(t, require, c)
|
|
resp := c.Query(t, indexName, `Extract(All())`)
|
|
m := resp.Results[0].(pilosa.ExtractedTable)
|
|
before := convert(m.Columns)
|
|
require.Equal([]uint64{0, 1, ShardWidth}, before, "these records are expected")
|
|
resp = c.Query(t, indexName, fmt.Sprintf(`Delete(ConstRow(columns=[%v]))`, ShardWidth))
|
|
require.NotNil(resp, "Response should not be nil")
|
|
require.NotEmpty(resp.Results)
|
|
require.Equal(true, resp.Results[0], "Change should have happened")
|
|
|
|
resp = c.Query(t, indexName, `Extract(All())`)
|
|
|
|
//Note none of the removed records should remain
|
|
m = resp.Results[0].(pilosa.ExtractedTable)
|
|
after := convert(m.Columns)
|
|
require.Equal([]uint64{0, 1}, after, "these records should be remaining")
|
|
})
|
|
|
|
// FB-1281: Delete() calls with an invalid bitmap filter would cause a panic.
|
|
// This test validates that the error is correctly propagated to the caller.
|
|
t.Run("DeleteWithBitmapError", func(t *testing.T) {
|
|
setup(t, require, c)
|
|
defer tearDown(t, require, c)
|
|
_, err := c.GetPrimary().API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: `Delete(Row(setfield == 1))`})
|
|
if err == nil || err.Error() != `executing: executeDelete: mapping on primary node: bsigroup not found` {
|
|
t.Fatalf("unexpected error: %s", err)
|
|
}
|
|
})
|
|
})
|
|
t.Run("DeleteRecordsBigWithRestart", func(t *testing.T) {
|
|
c := test.MustNewCluster(t, 1)
|
|
for _, n := range c.Nodes {
|
|
n.Config.Cluster.ReplicaN = 1
|
|
}
|
|
err := c.Start()
|
|
defer c.Close()
|
|
require.NoError(err, "Start cluster DeleteRecordsBig")
|
|
setupBig(t, require, c, 16)
|
|
defer tearDown(t, require, c)
|
|
node := c.GetNode(0)
|
|
resp := c.Query(t, indexName, `Delete(Row(setfield=12))`)
|
|
require.NotNil(resp, "Response should not be nil")
|
|
require.NotEmpty(resp.Results)
|
|
require.Equal(true, resp.Results[0], "Change should have happened")
|
|
resp = c.Query(t, indexName, `Count(Row(setfield=12))`)
|
|
require.NotNil(resp, "Response should not be nil")
|
|
require.NotEmpty(resp.Results)
|
|
require.Equal(uint64(0), resp.Results[0], "Should have removed")
|
|
err = node.Reopen()
|
|
require.NoError(err, "restart cluster DeleteRecordsBig")
|
|
err = c.AwaitState(disco.ClusterStateNormal, 10*time.Second)
|
|
require.NoError(err, "backToNormal")
|
|
})
|
|
}
|
|
|
|
func convert(before []pilosa.ExtractedTableColumn) []uint64 {
|
|
result := make([]uint64, 0)
|
|
for _, i := range before {
|
|
result = append(result, i.Column.ID)
|
|
}
|
|
return result
|
|
}
|
|
func convertKey(before []pilosa.ExtractedTableColumn) []string {
|
|
result := make([]string, 0)
|
|
for _, i := range before {
|
|
result = append(result, i.Column.Key)
|
|
}
|
|
return result
|
|
}
|