diff --git a/holder.go b/holder.go index bbd757ce4..e02daebb3 100644 --- a/holder.go +++ b/holder.go @@ -402,8 +402,8 @@ func (h *Holder) Field(index, name string) *Field { return idx.Field(name) } -// View returns the view for an index, field, and name. -func (h *Holder) View(index, field, name string) *View { +// view returns the view for an index, field, and name. +func (h *Holder) view(index, field, name string) *View { f := h.Field(index, field) if f == nil { return nil @@ -413,7 +413,7 @@ func (h *Holder) View(index, field, name string) *View { // Fragment returns the fragment for an index, field & shard. func (h *Holder) Fragment(index, field, view string, shard uint64) *Fragment { - v := h.View(index, field, view) + v := h.view(index, field, view) if v == nil { return nil } diff --git a/holder_internal_test.go b/holder_internal_test.go index 4bf9fae8b..fb725fe0a 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -18,6 +18,7 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" "strings" "testing" ) @@ -57,6 +58,43 @@ func newHolder() *tHolder { return h } +// MustCreateFieldIfNotExists returns a given field. Panic on error. +func (h *tHolder) MustCreateFieldIfNotExists(index, field string) *Field { + f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, FieldOptions{}) + if err != nil { + panic(err) + } + return f +} + +// MustCreateIndexIfNotExists returns a given index. Panic on error. +func (h *tHolder) MustCreateIndexIfNotExists(index string, opt IndexOptions) *Index { + idx, err := h.Holder.CreateIndexIfNotExists(index, opt) + if err != nil { + panic(err) + } + return idx +} + +// SetBit clears a bit on the given field. +func (h *tHolder) SetBit(index, field string, rowID, columnID uint64) { + f := h.MustCreateFieldIfNotExists(index, field) + _, err := f.SetBit(rowID, columnID, nil) + if err != nil { + panic(err) + } +} + +// Row returns a Row for a given field. +func (h *tHolder) Row(index, field string, rowID uint64) *Row { + f := h.MustCreateFieldIfNotExists(index, field) + row, err := f.Row(rowID) + if err != nil { + panic(err) + } + return row +} + func TestHolder_Optn(t *testing.T) { t.Run("ErrViewPermission", func(t *testing.T) { if os.Geteuid() == 0 { @@ -137,3 +175,117 @@ func TestHolder_Optn(t *testing.T) { }) } + +// Ensure holder can clean up orphaned fragments. +func TestHolderCleaner_CleanHolder(t *testing.T) { + cluster := NewTestCluster(2) + + // Create a local holder. + hldr0 := newHolder() + defer hldr0.Close() + + // Mock 2-node, fully replicated cluster. + cluster.ReplicaN = 2 + + cluster.Nodes[0].URI = NewTestURIFromHostPort("localhost", 0) + + // Create fields on nodes. + for _, hldr := range []*tHolder{hldr0} { + hldr.MustCreateFieldIfNotExists("i", "f") + hldr.MustCreateFieldIfNotExists("i", "f0") + hldr.MustCreateFieldIfNotExists("y", "z") + } + + // Set data on the local holder. + hldr0.SetBit("i", "f", 0, 10) + hldr0.SetBit("i", "f", 0, 4000) + hldr0.SetBit("i", "f", 2, 20) + hldr0.SetBit("i", "f", 3, 10) + hldr0.SetBit("i", "f", 120, 10) + hldr0.SetBit("i", "f", 200, 4) + + hldr0.SetBit("i", "f0", 9, ShardWidth+5) + + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4) + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5) + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7) + + // Set highest shard. + hldr0.Index("i").SetRemoteMaxShard(1) + hldr0.Index("y").SetRemoteMaxShard(2) + + // Keep replication the same and ensure we get the expected results. + cluster.ReplicaN = 2 + + // Set up cleaner for replication 2. + cleaner2 := HolderCleaner{ + Node: cluster.Nodes[0], + Holder: hldr0.Holder, + Cluster: cluster, + } + + if err := cleaner2.CleanHolder(); err != nil { + t.Fatal(err) + } + + // Verify data is the same on both nodes. + for i, hldr := range []*tHolder{hldr0} { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected columns(%d/0): %+v", i, a) + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected columns(%d/2): %+v", i, a) + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/3): %+v", i, a) + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/120): %+v", i, a) + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected columns(%d/200): %+v", i, a) + } + + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) { + t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) + } + + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { + t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) + } + } + + // Change replication factor to ensure we have fragments to remove. + cluster.ReplicaN = 1 + + // Set up cleaner for replication 1. + cleaner1 := HolderCleaner{ + Node: cluster.Nodes[0], + Holder: hldr0.Holder, + Cluster: cluster, + } + + if err := cleaner1.CleanHolder(); err != nil { + t.Fatal(err) + } + + // Verify data is the same on both nodes. + for i, hldr := range []*tHolder{hldr0} { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected columns(%d/0): %+v", i, a) + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected columns(%d/2): %+v", i, a) + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/3): %+v", i, a) + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/120): %+v", i, a) + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected columns(%d/200): %+v", i, a) + } + + f := hldr.Fragment("i", "f0", ViewStandard, 1) + if f != nil { + t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) + } + + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { + t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) + } + } +} diff --git a/holder_test.go b/holder_test.go index 421f16012..078ab55f3 100644 --- a/holder_test.go +++ b/holder_test.go @@ -362,117 +362,3 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { } } } - -// Ensure holder can clean up orphaned fragments. -func TestHolderCleaner_CleanHolder(t *testing.T) { - cluster := pilosa.NewTestCluster(2) - - // Create a local holder. - hldr0 := test.MustOpenHolder() - defer hldr0.Close() - - // Mock 2-node, fully replicated cluster. - cluster.ReplicaN = 2 - - cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0) - - // Create fields on nodes. - for _, hldr := range []*test.Holder{hldr0} { - hldr.MustCreateFieldIfNotExists("i", "f") - hldr.MustCreateFieldIfNotExists("i", "f0") - hldr.MustCreateFieldIfNotExists("y", "z") - } - - // Set data on the local holder. - hldr0.SetBit("i", "f", 0, 10) - hldr0.SetBit("i", "f", 0, 4000) - hldr0.SetBit("i", "f", 2, 20) - hldr0.SetBit("i", "f", 3, 10) - hldr0.SetBit("i", "f", 120, 10) - hldr0.SetBit("i", "f", 200, 4) - - hldr0.SetBit("i", "f0", 9, ShardWidth+5) - - hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4) - hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5) - hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7) - - // Set highest shard. - hldr0.Index("i").SetRemoteMaxShard(1) - hldr0.Index("y").SetRemoteMaxShard(2) - - // Keep replication the same and ensure we get the expected results. - cluster.ReplicaN = 2 - - // Set up cleaner for replication 2. - cleaner2 := pilosa.HolderCleaner{ - Node: cluster.Nodes[0], - Holder: hldr0.Holder, - Cluster: cluster, - } - - if err := cleaner2.CleanHolder(); err != nil { - t.Fatal(err) - } - - // Verify data is the same on both nodes. - for i, hldr := range []*test.Holder{hldr0} { - if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected columns(%d/200): %+v", i, a) - } - - if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) { - t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) - } - - if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { - t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) - } - } - - // Change replication factor to ensure we have fragments to remove. - cluster.ReplicaN = 1 - - // Set up cleaner for replication 1. - cleaner1 := pilosa.HolderCleaner{ - Node: cluster.Nodes[0], - Holder: hldr0.Holder, - Cluster: cluster, - } - - if err := cleaner1.CleanHolder(); err != nil { - t.Fatal(err) - } - - // Verify data is the same on both nodes. - for i, hldr := range []*test.Holder{hldr0} { - if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected columns(%d/200): %+v", i, a) - } - - f := hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - if f != nil { - t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) - } - - if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { - t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) - } - } -} diff --git a/http/client_test.go b/http/client_test.go index fc8e896b5..fb1105a27 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -306,7 +306,9 @@ func TestClient_FragmentBlocks(t *testing.T) { } // Verify data matches local blocks. - if a := hldr.Fragment("i", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) { + if a, err := cmd.API.FragmentBlocks(context.Background(), "i", "f", 0); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(a, blocks) { t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks)) } }