diff --git a/api.go b/api.go index a5d581b5b..bea6e1e76 100644 --- a/api.go +++ b/api.go @@ -388,7 +388,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldNa } // Retrieve view. - view, err := f.CreateViewIfNotExists(ViewStandard) + view, err := f.createViewIfNotExists(ViewStandard) if err != nil { return errors.Wrap(err, "creating view") } @@ -528,7 +528,7 @@ func (api *API) Views(ctx context.Context, indexName string, fieldName string) ( } // Fetch views. - views := f.Views() + views := f.views() return views, nil } @@ -545,7 +545,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri } // Delete the view. - if err := f.DeleteView(viewName); err != nil { + if err := f.deleteView(viewName); err != nil { // Ignore this error because views do not exist on all nodes due to shard distribution. if err != ErrInvalidView { return errors.Wrap(err, "deleting view") diff --git a/cluster.go b/cluster.go index dd7de8198..907d42c0d 100644 --- a/cluster.go +++ b/cluster.go @@ -618,7 +618,7 @@ func (c *cluster) fragsByHost(idx *Index) fragsByHost { fieldViews := make(viewsByField) for _, field := range idx.Fields() { - for _, view := range field.Views() { + for _, view := range field.views() { fieldViews.addView(field.Name(), view.name) } @@ -1222,7 +1222,7 @@ func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) err } // Create view. - v, err := f.CreateViewIfNotExists(src.View) + v, err := f.createViewIfNotExists(src.View) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 845fdd888..09ccb9915 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -706,7 +706,7 @@ func TestCluster_ResizeStates(t *testing.T) { // Before starting the resize, get the CheckSum to use for // comparison later. node0Field := node0.holder.Field("i", "f") - node0View := node0Field.View("standard") + node0View := node0Field.view("standard") node0Fragment := node0View.Fragment(1) node0Checksum := node0Fragment.Checksum() @@ -735,7 +735,7 @@ func TestCluster_ResizeStates(t *testing.T) { // Bits // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. node1Field := node1.holder.Field("i", "f") - node1View := node1Field.View("standard") + node1View := node1Field.view("standard") node1Fragment := node1View.Fragment(1) // Ensure checksums are the same. diff --git a/executor_test.go b/executor_test.go index 09535c0cd..4e708fd62 100644 --- a/executor_test.go +++ b/executor_test.go @@ -531,9 +531,10 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() + err := c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) @@ -571,7 +572,10 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + err := c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) @@ -664,9 +668,10 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr.SetBit("i", "other", 100, ShardWidth+1) hldr.SetBit("i", "other", 100, ShardWidth+2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache() + err := c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } // Execute query. if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(other=100), n=3)`}); err != nil { diff --git a/field.go b/field.go index d68b26971..090e5926d 100644 --- a/field.go +++ b/field.go @@ -59,7 +59,7 @@ type Field struct { index string name string - views map[string]*View + viewMap map[string]*View // Row attribute storage and cache rowAttrStore AttrStore @@ -131,7 +131,7 @@ func NewField(path, index, name string, options FieldOptions) (*Field, error) { index: index, name: name, - views: make(map[string]*View), + viewMap: make(map[string]*View), rowAttrStore: nopStore, @@ -163,7 +163,7 @@ func (f *Field) MaxShard() uint64 { defer f.mu.RUnlock() var max uint64 - for _, view := range f.views { + for _, view := range f.viewMap { if viewMaxShard := view.calculateMaxShard(); viewMaxShard > max { max = viewMaxShard } @@ -280,7 +280,7 @@ func (f *Field) openViews() error { return fmt.Errorf("opening view: view=%s, err=%s", view.name, err) } view.RowAttrStore = f.rowAttrStore - f.views[view.name] = view + f.viewMap[view.name] = view } return nil @@ -399,12 +399,12 @@ func (f *Field) Close() error { } // Close all views. - for _, view := range f.views { + for _, view := range f.viewMap { if err := view.close(); err != nil { return err } } - f.views = make(map[string]*View) + f.viewMap = make(map[string]*View) return nil } @@ -482,8 +482,8 @@ func (f *Field) deleteBSIGroupAndView(name string) error { // Remove views. viewName := viewBSIGroupPrefix + name - if view := f.views[viewName]; view != nil { - delete(f.views, viewName) + if view := f.viewMap[viewName]; view != nil { + delete(f.viewMap, viewName) if err := view.close(); err != nil { return errors.Wrap(err, "closing view") @@ -540,22 +540,22 @@ func (f *Field) ViewPath(name string) string { return filepath.Join(f.path, "views", name) } -// View returns a view in the field by name. -func (f *Field) View(name string) *View { +// view returns a view in the field by name. +func (f *Field) view(name string) *View { f.mu.RLock() defer f.mu.RUnlock() - return f.view(name) + return f.unprotectedView(name) } -func (f *Field) view(name string) *View { return f.views[name] } +func (f *Field) unprotectedView(name string) *View { return f.viewMap[name] } -// Views returns a list of all views in the field. -func (f *Field) Views() []*View { +// views returns a list of all views in the field. +func (f *Field) views() []*View { f.mu.RLock() defer f.mu.RUnlock() - other := make([]*View, 0, len(f.views)) - for _, view := range f.views { + other := make([]*View, 0, len(f.viewMap)) + for _, view := range f.viewMap { other = append(other, view) } return other @@ -566,8 +566,8 @@ func (f *Field) viewNames() []string { f.mu.Lock() defer f.mu.Unlock() - other := make([]string, 0, len(f.views)) - for viewName, _ := range f.views { + other := make([]string, 0, len(f.viewMap)) + for viewName, _ := range f.viewMap { other = append(other, viewName) } return other @@ -575,14 +575,14 @@ func (f *Field) viewNames() []string { // RecalculateCaches recalculates caches on every view in the field. func (f *Field) RecalculateCaches() { - for _, view := range f.Views() { + for _, view := range f.views() { view.recalculateCaches() } } -// CreateViewIfNotExists returns the named view, creating it if necessary. +// createViewIfNotExists returns the named view, creating it if necessary. // Additionally, a CreateViewMessage is sent to the cluster. -func (f *Field) CreateViewIfNotExists(name string) (*View, error) { +func (f *Field) createViewIfNotExists(name string) (*View, error) { view, created, err := f.createViewIfNotExistsBase(name) if err != nil { return nil, err @@ -610,7 +610,7 @@ func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) { f.mu.Lock() defer f.mu.Unlock() - if view := f.views[name]; view != nil { + if view := f.viewMap[name]; view != nil { return view, false, nil } view := f.newView(f.ViewPath(name), name) @@ -619,7 +619,7 @@ func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) { return nil, false, errors.Wrap(err, "opening view") } view.RowAttrStore = f.rowAttrStore - f.views[view.name] = view + f.viewMap[view.name] = view return view, true, nil } @@ -634,9 +634,9 @@ func (f *Field) newView(path, name string) *View { return view } -// DeleteView removes the view from the field. -func (f *Field) DeleteView(name string) error { - view := f.views[name] +// deleteView removes the view from the field. +func (f *Field) deleteView(name string) error { + view := f.viewMap[name] if view == nil { return ErrInvalidView } @@ -651,7 +651,7 @@ func (f *Field) DeleteView(name string) error { return errors.Wrap(err, "deleting directory") } - delete(f.views, name) + delete(f.viewMap, name) return nil } @@ -661,7 +661,7 @@ func (f *Field) Row(rowID uint64) (*Row, error) { if f.Type() != FieldTypeSet { return nil, errors.Errorf("row method unsupported for field type: %s", f.Type()) } - view := f.View(ViewStandard) + view := f.view(ViewStandard) if view == nil { return nil, ErrInvalidView } @@ -671,7 +671,7 @@ func (f *Field) Row(rowID uint64) (*Row, error) { // ViewRow returns a row for a view and shard. // TODO: unexport this with views (it's only used in tests). func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) { - view := f.View(viewName) + view := f.view(viewName) if view == nil { return nil, ErrInvalidView } @@ -683,7 +683,7 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err viewName := ViewStandard // Retrieve view. Exit if it doesn't exist. - view, err := f.CreateViewIfNotExists(viewName) + view, err := f.createViewIfNotExists(viewName) if err != nil { return changed, errors.Wrap(err, "creating view") } @@ -702,7 +702,7 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err // If a timestamp is specified then set bits across all views for the quantum. for _, subname := range viewsByTime(viewName, *t, f.TimeQuantum()) { - view, err := f.CreateViewIfNotExists(subname) + view, err := f.createViewIfNotExists(subname) if err != nil { return changed, errors.Wrapf(err, "creating view %s", subname) } @@ -722,7 +722,7 @@ func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) { viewName := ViewStandard // Retrieve view. Exit if it doesn't exist. - view, present := f.views[viewName] + view, present := f.viewMap[viewName] if !present { return changed, errors.Wrap(err, "clearing missing view") @@ -734,7 +734,7 @@ func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) { } else if v { changed = v } - if len(f.views) == 1 { // assuming no time views + if len(f.viewMap) == 1 { // assuming no time views return changed, nil } lastViewNameSize := 0 @@ -774,11 +774,11 @@ func groupCompare(a, b string, offset int) (lt, eq bool) { } func (f *Field) allTimeViewsSortedByQuantum() (me []*View) { - me = make([]*View, len(f.views), len(f.views)) + me = make([]*View, len(f.viewMap), len(f.viewMap)) prefix := ViewStandard + "_" offset := len(ViewStandard) + 1 i := 0 - for _, v := range f.views { + for _, v := range f.viewMap { if len(v.name) > offset && strings.Compare(v.name[:offset], prefix) == 0 { // skip non-time views me[i] = v i++ @@ -811,7 +811,7 @@ func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { } // Fetch target view. - view := f.View(viewBSIGroupPrefix + f.name) + view := f.view(viewBSIGroupPrefix + f.name) if view == nil { return 0, false, nil } @@ -838,7 +838,7 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) } // Fetch target view. - view, err := f.CreateViewIfNotExists(viewBSIGroupPrefix + f.name) + view, err := f.createViewIfNotExists(viewBSIGroupPrefix + f.name) if err != nil { return false, errors.Wrap(err, "creating view") } @@ -857,7 +857,7 @@ func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) { return 0, 0, ErrBSIGroupNotFound } - view := f.View(viewBSIGroupPrefix + name) + view := f.view(viewBSIGroupPrefix + name) if view == nil { return 0, 0, nil } @@ -877,7 +877,7 @@ func (f *Field) Min(filter *Row, name string) (min, count int64, err error) { return 0, 0, ErrBSIGroupNotFound } - view := f.View(viewBSIGroupPrefix + name) + view := f.view(viewBSIGroupPrefix + name) if view == nil { return 0, 0, nil } @@ -897,7 +897,7 @@ func (f *Field) Max(filter *Row, name string) (max, count int64, err error) { return 0, 0, ErrBSIGroupNotFound } - view := f.View(viewBSIGroupPrefix + name) + view := f.view(viewBSIGroupPrefix + name) if view == nil { return 0, 0, nil } @@ -919,7 +919,7 @@ func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) } // Retrieve bsiGroup's view. - view := f.View(viewBSIGroupPrefix + name) + view := f.view(viewBSIGroupPrefix + name) if view == nil { return nil, nil } @@ -942,7 +942,7 @@ func (f *Field) RangeBetween(name string, predicateMin, predicateMax int64) (*Ro } // Retrieve bsiGroup's view. - view := f.View(viewBSIGroupPrefix + name) + view := f.view(viewBSIGroupPrefix + name) if view == nil { return nil, nil } @@ -994,7 +994,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // Import into each fragment. for key, data := range dataByFragment { - view, err := f.CreateViewIfNotExists(key.View) + view, err := f.createViewIfNotExists(key.View) if err != nil { return errors.Wrap(err, "creating view") } @@ -1046,7 +1046,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { // The view must already exist (i.e. we can't create it) // because we need to know bitDepth (based on min/max value). - view, err := f.CreateViewIfNotExists(key.View) + view, err := f.createViewIfNotExists(key.View) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/field_internal_test.go b/field_internal_test.go index 91b1af02c..176a4d0e8 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -15,6 +15,8 @@ package pilosa import ( + "io/ioutil" + "os" "reflect" "testing" @@ -147,3 +149,133 @@ func TestBSIGroup_BaseValue(t *testing.T) { } }) } + +// Ensure field can open and retrieve a view. +func TestField_DeleteView(t *testing.T) { + f := MustOpenField(FieldOptions{}) + defer f.Close() + + viewName := ViewStandard + "_v" + + // Create view. + view, err := f.createViewIfNotExists(viewName) + if err != nil { + t.Fatal(err) + } else if view == nil { + t.Fatal("expected view") + } + + err = f.deleteView(viewName) + if err != nil { + t.Fatal(err) + } + + if f.view(viewName) != nil { + t.Fatal("view still exists in field") + } + + // Recreate view with same name, verify that the old view was not reused. + view2, err := f.createViewIfNotExists(viewName) + if err != nil { + t.Fatal(err) + } else if view == view2 { + t.Fatal("failed to create new view") + } +} + +// TestField represents a test wrapper for Field. +type TestField struct { + *Field +} + +// NewTestField returns a new instance of TestField d/0. +func NewTestField(options FieldOptions) *TestField { + path, err := ioutil.TempDir("", "pilosa-field-") + if err != nil { + panic(err) + } + field, err := NewField(path, "i", "f", options) + if err != nil { + panic(err) + } + return &TestField{Field: field} +} + +// MustOpenField returns a new, opened field at a temporary path. Panic on error. +func MustOpenField(options FieldOptions) *TestField { + f := NewTestField(options) + if err := f.Open(); err != nil { + panic(err) + } + return f +} + +// Close closes the field and removes the underlying data. +func (f *TestField) Close() error { + defer os.RemoveAll(f.Path()) + return f.Field.Close() +} + +// Reopen closes the index and reopens it. +func (f *TestField) Reopen() error { + var err error + if err := f.Field.Close(); err != nil { + return err + } + + path, index, name := f.Path(), f.Index(), f.Name() + f.Field, err = NewField(path, index, name, FieldOptions{}) + if err != nil { + return err + } + + if err := f.Open(); err != nil { + return err + } + return nil +} + +// Ensure field can open and retrieve a view. +func TestField_CreateViewIfNotExists(t *testing.T) { + f := MustOpenField(FieldOptions{}) + defer f.Close() + + // Create view. + view, err := f.createViewIfNotExists("v") + if err != nil { + t.Fatal(err) + } else if view == nil { + t.Fatal("expected view") + } + + // Retrieve existing view. + view2, err := f.createViewIfNotExists("v") + if err != nil { + t.Fatal(err) + } else if view != view2 { + t.Fatal("view mismatch") + } + + if view != f.view("v") { + t.Fatal("view mismatch") + } +} + +func TestField_SetTimeQuantum(t *testing.T) { + f := MustOpenField(FieldOptions{Type: FieldTypeTime}) + defer f.Close() + + // Set & retrieve time quantum. + if err := f.SetTimeQuantum(TimeQuantum("YMDH")); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %s", q) + } + + // Reload field and verify that it is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum (reopen): %s", q) + } +} diff --git a/field_test.go b/field_test.go index e3719a35a..4b1def96c 100644 --- a/field_test.go +++ b/field_test.go @@ -22,52 +22,6 @@ import ( "github.com/pilosa/pilosa/test" ) -// Ensure field can open and retrieve a view. -func TestField_CreateViewIfNotExists(t *testing.T) { - f := test.MustOpenField(pilosa.FieldOptions{}) - defer f.Close() - - // Create view. - view, err := f.CreateViewIfNotExists("v") - if err != nil { - t.Fatal(err) - } else if view == nil { - t.Fatal("expected view") - } - - // Retrieve existing view. - view2, err := f.CreateViewIfNotExists("v") - if err != nil { - t.Fatal(err) - } else if view != view2 { - t.Fatal("view mismatch") - } - - if view != f.View("v") { - t.Fatal("view mismatch") - } -} - -// Ensure field can set its time quantum. -func TestField_SetTimeQuantum(t *testing.T) { - f := test.MustOpenField(pilosa.FieldOptions{Type: pilosa.FieldTypeTime}) - defer f.Close() - - // Set & retrieve time quantum. - if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { - t.Fatal(err) - } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum: %s", q) - } - - // Reload field and verify that it is persisted. - if err := f.Reopen(); err != nil { - t.Fatal(err) - } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum (reopen): %s", q) - } -} - // Ensure a field can set & read a bsiGroup value. func TestField_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { @@ -249,36 +203,3 @@ func TestField_NameValidation(t *testing.T) { } } } - -// Ensure field can open and retrieve a view. -func TestField_DeleteView(t *testing.T) { - f := test.MustOpenField(pilosa.FieldOptions{}) - defer f.Close() - - viewName := pilosa.ViewStandard + "_v" - - // Create view. - view, err := f.CreateViewIfNotExists(viewName) - if err != nil { - t.Fatal(err) - } else if view == nil { - t.Fatal("expected view") - } - - err = f.DeleteView(viewName) - if err != nil { - t.Fatal(err) - } - - if f.View(viewName) != nil { - t.Fatal("view still exists in field") - } - - // Recreate view with same name, verify that the old view was not reused. - view2, err := f.CreateViewIfNotExists(viewName) - if err != nil { - t.Fatal(err) - } else if view == view2 { - t.Fatal("failed to create new view") - } -} diff --git a/fragment_internal_test.go b/fragment_internal_test.go index c779dd596..c73889698 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -756,7 +756,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } // Create view. - view, err := field.CreateViewIfNotExists(ViewStandard) + view, err := field.createViewIfNotExists(ViewStandard) if err != nil { t.Fatal(err) } @@ -922,7 +922,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { } // Create view. - view, err := field.CreateViewIfNotExists(ViewStandard) + view, err := field.createViewIfNotExists(ViewStandard) if err != nil { t.Fatal(err) } @@ -953,7 +953,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { } // Re-fetch fragment. - f = index.Field("f").View(ViewStandard).Fragment(0) + f = index.Field("f").view(ViewStandard).Fragment(0) // Re-verify correct cache type and size. if cache, ok := f.cache.(*RankCache); !ok { diff --git a/holder.go b/holder.go index f050ad896..bbd757ce4 100644 --- a/holder.go +++ b/holder.go @@ -216,7 +216,7 @@ func (h *Holder) Schema() []*IndexInfo { di := &IndexInfo{Name: index.Name()} for _, field := range index.Fields() { fi := &FieldInfo{Name: field.Name(), Options: field.Options()} - for _, view := range field.Views() { + for _, view := range field.views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } sort.Sort(viewInfoSlice(fi.Views)) @@ -247,7 +247,7 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { } // Create views that don't exist. for _, v := range f.Views { - _, err := field.CreateViewIfNotExists(v) + _, err := field.createViewIfNotExists(v) if err != nil { return errors.Wrap(err, "creating view") } @@ -408,7 +408,7 @@ func (h *Holder) View(index, field, name string) *View { if f == nil { return nil } - return f.View(name) + return f.view(name) } // Fragment returns the fragment for an index, field & shard. @@ -439,7 +439,7 @@ func (h *Holder) monitorCacheFlush() { func (h *Holder) flushCaches() { for _, index := range h.Indexes() { for _, field := range index.Fields() { - for _, view := range field.Views() { + for _, view := range field.views() { for _, fragment := range view.allFragments() { select { case <-h.closing: @@ -748,7 +748,7 @@ func (s *HolderSyncer) syncFragment(index, field, view string, shard uint64) err } // Ensure view exists locally. - v, err := f.CreateViewIfNotExists(view) + v, err := f.createViewIfNotExists(view) if err != nil { return errors.Wrap(err, "creating view") } @@ -808,7 +808,7 @@ func (c *HolderCleaner) CleanHolder() error { // Get the fragments registered in memory. for _, field := range index.Fields() { - for _, view := range field.Views() { + for _, view := range field.views() { for _, fragment := range view.allFragments() { fragShard := fragment.shard // Ignore fragments that should be present. diff --git a/holder_internal_test.go b/holder_internal_test.go new file mode 100644 index 000000000..4bf9fae8b --- /dev/null +++ b/holder_internal_test.go @@ -0,0 +1,139 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" +) + +type tHolder struct { + *Holder +} + +// Close closes the holder and removes all underlying data. +func (h *tHolder) Close() error { + defer os.RemoveAll(h.Path) + return h.Holder.Close() +} + +// Reopen instantiates and opens a new holder. +// Note that the holder must be Closed first. +func (h *tHolder) Reopen() error { + path, logger := h.Path, h.Holder.Logger + h.Holder = NewHolder() + h.Holder.Path = path + h.Holder.Logger = logger + if err := h.Holder.Open(); err != nil { + return err + } + + return nil +} + +func newHolder() *tHolder { + path, err := ioutil.TempDir("", "pilosa-") + if err != nil { + panic(err) + } + + h := &tHolder{Holder: NewHolder()} + h.Path = path + return h +} + +func TestHolder_Optn(t *testing.T) { + t.Run("ErrViewPermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } + h := newHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { + t.Fatal(err) + } else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := field.createViewIfNotExists(ViewStandard); err != nil { + t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrViewFragmentsMkdir", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } + h := newHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { + t.Fatal(err) + } else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := field.createViewIfNotExists(ViewStandard); err != nil { + t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFragmentCachePermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } + h := newHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { + t.Fatal(err) + } else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil { + t.Fatal(err) + } else if view, err := field.createViewIfNotExists(ViewStandard); err != nil { + t.Fatal(err) + } else if _, err := field.SetBit(0, 0, nil); err != nil { + t.Fatal(err) + } else if err := view.Fragment(0).FlushCache(); err != nil { + t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + +} diff --git a/holder_test.go b/holder_test.go index fe6a1a9f8..421f16012 100644 --- a/holder_test.go +++ b/holder_test.go @@ -148,55 +148,6 @@ func TestHolder_Open(t *testing.T) { } }) - t.Run("ErrViewPermission", func(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("Skipping permissions test since user is root.") - } - h := test.MustOpenHolder() - defer h.Close() - - if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil { - t.Fatal(err) - } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777) - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { - t.Fatalf("unexpected error: %s", err) - } - }) - t.Run("ErrViewFragmentsMkdir", func(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("Skipping permissions test since user is root.") - } - h := test.MustOpenHolder() - defer h.Close() - - if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil { - t.Fatal(err) - } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777) - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { - t.Fatalf("unexpected error: %s", err) - } - }) - t.Run("ErrFragmentStoragePermission", func(t *testing.T) { if os.Geteuid() == 0 { t.Skip("Skipping permissions test since user is root.") @@ -242,34 +193,6 @@ func TestHolder_Open(t *testing.T) { } }) - t.Run("ErrFragmentCachePermission", func(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("Skipping permissions test since user is root.") - } - h := test.MustOpenHolder() - defer h.Close() - - if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if _, err := field.SetBit(0, 0, nil); err != nil { - t.Fatal(err) - } else if err := view.Fragment(0).FlushCache(); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil { - t.Fatal(err) - } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666) - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { - t.Fatalf("unexpected error: %s", err) - } - }) } func TestHolder_HasData(t *testing.T) { diff --git a/server.go b/server.go index fe99abf4c..d14e3c37d 100644 --- a/server.go +++ b/server.go @@ -479,7 +479,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if f == nil { return fmt.Errorf("Local Field not found: %s", obj.Field) } - err := f.DeleteView(obj.View) + err := f.deleteView(obj.View) if err != nil { return err } diff --git a/test/holder.go b/test/holder.go index 6cdd57f55..d313c037d 100644 --- a/test/holder.go +++ b/test/holder.go @@ -89,24 +89,6 @@ func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field { return f } -// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. -func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, shard uint64) *Fragment { - idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) - if err != nil { - panic(err) - } - v, err := f.CreateViewIfNotExists(view) - if err != nil { - panic(err) - } - frag, err := v.CreateFragmentIfNotExists(shard) - if err != nil { - panic(err) - } - return &Fragment{Fragment: frag} -} - // Row returns a Row for a given field. func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) diff --git a/utils_internal_test.go b/utils_internal_test.go index 1d88c3d32..939e8e04c 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -373,7 +373,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi if destFragment == nil { // Create fragment on destination if it doesn't exist. f := destCluster.holder.Field(src.Index, src.Field) - v := f.View(src.View) + v := f.view(src.View) var err error destFragment, err = v.CreateFragmentIfNotExists(src.Shard) if err != nil {