mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
unexport Holder.view and prepare to unexport Holder.Fragment
This commit is contained in:
Matt Jaffee
parent
1f3d22b5f6
commit
a6a0c6a7c3
4 changed files with 158 additions and 118 deletions
|
|
@ -402,8 +402,8 @@ func (h *Holder) Field(index, name string) *Field {
|
|||
return idx.Field(name)
|
||||
}
|
||||
|
||||
// View returns the view for an index, field, and name.
|
||||
func (h *Holder) View(index, field, name string) *View {
|
||||
// view returns the view for an index, field, and name.
|
||||
func (h *Holder) view(index, field, name string) *View {
|
||||
f := h.Field(index, field)
|
||||
if f == nil {
|
||||
return nil
|
||||
|
|
@ -413,7 +413,7 @@ func (h *Holder) View(index, field, name string) *View {
|
|||
|
||||
// Fragment returns the fragment for an index, field & shard.
|
||||
func (h *Holder) Fragment(index, field, view string, shard uint64) *Fragment {
|
||||
v := h.View(index, field, view)
|
||||
v := h.view(index, field, view)
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import (
|
|||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
|
@ -57,6 +58,43 @@ func newHolder() *tHolder {
|
|||
return h
|
||||
}
|
||||
|
||||
// MustCreateFieldIfNotExists returns a given field. Panic on error.
|
||||
func (h *tHolder) MustCreateFieldIfNotExists(index, field string) *Field {
|
||||
f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, FieldOptions{})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// MustCreateIndexIfNotExists returns a given index. Panic on error.
|
||||
func (h *tHolder) MustCreateIndexIfNotExists(index string, opt IndexOptions) *Index {
|
||||
idx, err := h.Holder.CreateIndexIfNotExists(index, opt)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// SetBit clears a bit on the given field.
|
||||
func (h *tHolder) SetBit(index, field string, rowID, columnID uint64) {
|
||||
f := h.MustCreateFieldIfNotExists(index, field)
|
||||
_, err := f.SetBit(rowID, columnID, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Row returns a Row for a given field.
|
||||
func (h *tHolder) Row(index, field string, rowID uint64) *Row {
|
||||
f := h.MustCreateFieldIfNotExists(index, field)
|
||||
row, err := f.Row(rowID)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
func TestHolder_Optn(t *testing.T) {
|
||||
t.Run("ErrViewPermission", func(t *testing.T) {
|
||||
if os.Geteuid() == 0 {
|
||||
|
|
@ -137,3 +175,117 @@ func TestHolder_Optn(t *testing.T) {
|
|||
})
|
||||
|
||||
}
|
||||
|
||||
// Ensure holder can clean up orphaned fragments.
|
||||
func TestHolderCleaner_CleanHolder(t *testing.T) {
|
||||
cluster := NewTestCluster(2)
|
||||
|
||||
// Create a local holder.
|
||||
hldr0 := newHolder()
|
||||
defer hldr0.Close()
|
||||
|
||||
// Mock 2-node, fully replicated cluster.
|
||||
cluster.ReplicaN = 2
|
||||
|
||||
cluster.Nodes[0].URI = NewTestURIFromHostPort("localhost", 0)
|
||||
|
||||
// Create fields on nodes.
|
||||
for _, hldr := range []*tHolder{hldr0} {
|
||||
hldr.MustCreateFieldIfNotExists("i", "f")
|
||||
hldr.MustCreateFieldIfNotExists("i", "f0")
|
||||
hldr.MustCreateFieldIfNotExists("y", "z")
|
||||
}
|
||||
|
||||
// Set data on the local holder.
|
||||
hldr0.SetBit("i", "f", 0, 10)
|
||||
hldr0.SetBit("i", "f", 0, 4000)
|
||||
hldr0.SetBit("i", "f", 2, 20)
|
||||
hldr0.SetBit("i", "f", 3, 10)
|
||||
hldr0.SetBit("i", "f", 120, 10)
|
||||
hldr0.SetBit("i", "f", 200, 4)
|
||||
|
||||
hldr0.SetBit("i", "f0", 9, ShardWidth+5)
|
||||
|
||||
hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4)
|
||||
hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5)
|
||||
hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7)
|
||||
|
||||
// Set highest shard.
|
||||
hldr0.Index("i").SetRemoteMaxShard(1)
|
||||
hldr0.Index("y").SetRemoteMaxShard(2)
|
||||
|
||||
// Keep replication the same and ensure we get the expected results.
|
||||
cluster.ReplicaN = 2
|
||||
|
||||
// Set up cleaner for replication 2.
|
||||
cleaner2 := HolderCleaner{
|
||||
Node: cluster.Nodes[0],
|
||||
Holder: hldr0.Holder,
|
||||
Cluster: cluster,
|
||||
}
|
||||
|
||||
if err := cleaner2.CleanHolder(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify data is the same on both nodes.
|
||||
for i, hldr := range []*tHolder{hldr0} {
|
||||
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
|
||||
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
|
||||
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
|
||||
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
|
||||
}
|
||||
|
||||
if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) {
|
||||
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
|
||||
}
|
||||
|
||||
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
|
||||
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
|
||||
}
|
||||
}
|
||||
|
||||
// Change replication factor to ensure we have fragments to remove.
|
||||
cluster.ReplicaN = 1
|
||||
|
||||
// Set up cleaner for replication 1.
|
||||
cleaner1 := HolderCleaner{
|
||||
Node: cluster.Nodes[0],
|
||||
Holder: hldr0.Holder,
|
||||
Cluster: cluster,
|
||||
}
|
||||
|
||||
if err := cleaner1.CleanHolder(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify data is the same on both nodes.
|
||||
for i, hldr := range []*tHolder{hldr0} {
|
||||
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
|
||||
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
|
||||
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
|
||||
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
|
||||
}
|
||||
|
||||
f := hldr.Fragment("i", "f0", ViewStandard, 1)
|
||||
if f != nil {
|
||||
t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
|
||||
}
|
||||
|
||||
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
|
||||
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
114
holder_test.go
114
holder_test.go
|
|
@ -362,117 +362,3 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure holder can clean up orphaned fragments.
|
||||
func TestHolderCleaner_CleanHolder(t *testing.T) {
|
||||
cluster := pilosa.NewTestCluster(2)
|
||||
|
||||
// Create a local holder.
|
||||
hldr0 := test.MustOpenHolder()
|
||||
defer hldr0.Close()
|
||||
|
||||
// Mock 2-node, fully replicated cluster.
|
||||
cluster.ReplicaN = 2
|
||||
|
||||
cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0)
|
||||
|
||||
// Create fields on nodes.
|
||||
for _, hldr := range []*test.Holder{hldr0} {
|
||||
hldr.MustCreateFieldIfNotExists("i", "f")
|
||||
hldr.MustCreateFieldIfNotExists("i", "f0")
|
||||
hldr.MustCreateFieldIfNotExists("y", "z")
|
||||
}
|
||||
|
||||
// Set data on the local holder.
|
||||
hldr0.SetBit("i", "f", 0, 10)
|
||||
hldr0.SetBit("i", "f", 0, 4000)
|
||||
hldr0.SetBit("i", "f", 2, 20)
|
||||
hldr0.SetBit("i", "f", 3, 10)
|
||||
hldr0.SetBit("i", "f", 120, 10)
|
||||
hldr0.SetBit("i", "f", 200, 4)
|
||||
|
||||
hldr0.SetBit("i", "f0", 9, ShardWidth+5)
|
||||
|
||||
hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4)
|
||||
hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5)
|
||||
hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7)
|
||||
|
||||
// Set highest shard.
|
||||
hldr0.Index("i").SetRemoteMaxShard(1)
|
||||
hldr0.Index("y").SetRemoteMaxShard(2)
|
||||
|
||||
// Keep replication the same and ensure we get the expected results.
|
||||
cluster.ReplicaN = 2
|
||||
|
||||
// Set up cleaner for replication 2.
|
||||
cleaner2 := pilosa.HolderCleaner{
|
||||
Node: cluster.Nodes[0],
|
||||
Holder: hldr0.Holder,
|
||||
Cluster: cluster,
|
||||
}
|
||||
|
||||
if err := cleaner2.CleanHolder(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify data is the same on both nodes.
|
||||
for i, hldr := range []*test.Holder{hldr0} {
|
||||
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
|
||||
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
|
||||
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
|
||||
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
|
||||
}
|
||||
|
||||
if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) {
|
||||
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
|
||||
}
|
||||
|
||||
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
|
||||
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
|
||||
}
|
||||
}
|
||||
|
||||
// Change replication factor to ensure we have fragments to remove.
|
||||
cluster.ReplicaN = 1
|
||||
|
||||
// Set up cleaner for replication 1.
|
||||
cleaner1 := pilosa.HolderCleaner{
|
||||
Node: cluster.Nodes[0],
|
||||
Holder: hldr0.Holder,
|
||||
Cluster: cluster,
|
||||
}
|
||||
|
||||
if err := cleaner1.CleanHolder(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify data is the same on both nodes.
|
||||
for i, hldr := range []*test.Holder{hldr0} {
|
||||
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
|
||||
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
|
||||
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
|
||||
} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
|
||||
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
|
||||
}
|
||||
|
||||
f := hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
|
||||
if f != nil {
|
||||
t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
|
||||
}
|
||||
|
||||
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
|
||||
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -306,7 +306,9 @@ func TestClient_FragmentBlocks(t *testing.T) {
|
|||
}
|
||||
|
||||
// Verify data matches local blocks.
|
||||
if a := hldr.Fragment("i", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) {
|
||||
if a, err := cmd.API.FragmentBlocks(context.Background(), "i", "f", 0); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(a, blocks) {
|
||||
t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue