scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1443 from jaffee/field-unexport

Browse source 
work on unexporting View stuff
This commit is contained in:
Matthew Jaffee 2018-07-02 10:38:11 -05:00 • committed by GitHub
commit 7bd55f47e7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 345 additions and 243 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
api.go
View file

@ -388,7 +388,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldNa
}
// Retrieve view.
view, err := f.CreateViewIfNotExists(ViewStandard)
view, err := f.createViewIfNotExists(ViewStandard)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -528,7 +528,7 @@ func (api *API) Views(ctx context.Context, indexName string, fieldName string) (
}
// Fetch views.
views := f.Views()
views := f.views()
return views, nil
}
@ -545,7 +545,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri
}
// Delete the view.
if err := f.DeleteView(viewName); err != nil {
if err := f.deleteView(viewName); err != nil {
// Ignore this error because views do not exist on all nodes due to shard distribution.
if err != ErrInvalidView {
return errors.Wrap(err, "deleting view")

4
cluster.go
View file

@ -618,7 +618,7 @@ func (c *cluster) fragsByHost(idx *Index) fragsByHost {
fieldViews := make(viewsByField)
for _, field := range idx.Fields() {
for _, view := range field.Views() {
for _, view := range field.views() {
fieldViews.addView(field.Name(), view.name)
}
@ -1222,7 +1222,7 @@ func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) err
}
// Create view.
v, err := f.CreateViewIfNotExists(src.View)
v, err := f.createViewIfNotExists(src.View)
if err != nil {
return errors.Wrap(err, "creating view")
}

4
cluster_internal_test.go
View file

@ -706,7 +706,7 @@ func TestCluster_ResizeStates(t *testing.T) {
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Field := node0.holder.Field("i", "f")
node0View := node0Field.View("standard")
node0View := node0Field.view("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
@ -735,7 +735,7 @@ func TestCluster_ResizeStates(t *testing.T) {
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Field := node1.holder.Field("i", "f")
node1View := node1Field.View("standard")
node1View := node1Field.view("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.

19
executor_test.go
View file

@ -531,9 +531,10 @@ func TestExecutor_Execute_TopN(t *testing.T) {
t.Fatal(err)
}
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
err := c[0].RecalculateCaches()
if err != nil {
t.Fatalf("recalculating caches: %v", err)
}
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil {
t.Fatal(err)
@ -571,7 +572,10 @@ func TestExecutor_Execute_TopN(t *testing.T) {
t.Fatal(err)
}
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
err := c[0].RecalculateCaches()
if err != nil {
t.Fatalf("recalculating caches: %v", err)
}
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil {
t.Fatal(err)
@ -664,9 +668,10 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
hldr.SetBit("i", "other", 100, ShardWidth+1)
hldr.SetBit("i", "other", 100, ShardWidth+2)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache()
err := c[0].RecalculateCaches()
if err != nil {
t.Fatalf("recalculating caches: %v", err)
}
// Execute query.
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(other=100), n=3)`}); err != nil {

88
field.go
View file

@ -59,7 +59,7 @@ type Field struct {
index string
name string
views map[string]*View
viewMap map[string]*View
// Row attribute storage and cache
rowAttrStore AttrStore
@ -131,7 +131,7 @@ func NewField(path, index, name string, options FieldOptions) (*Field, error) {
index: index,
name: name,
views: make(map[string]*View),
viewMap: make(map[string]*View),
rowAttrStore: nopStore,
@ -163,7 +163,7 @@ func (f *Field) MaxShard() uint64 {
defer f.mu.RUnlock()
var max uint64
for _, view := range f.views {
for _, view := range f.viewMap {
if viewMaxShard := view.calculateMaxShard(); viewMaxShard > max {
max = viewMaxShard
}
@ -280,7 +280,7 @@ func (f *Field) openViews() error {
return fmt.Errorf("opening view: view=%s, err=%s", view.name, err)
}
view.RowAttrStore = f.rowAttrStore
f.views[view.name] = view
f.viewMap[view.name] = view
}
return nil
@ -399,12 +399,12 @@ func (f *Field) Close() error {
}
// Close all views.
for _, view := range f.views {
for _, view := range f.viewMap {
if err := view.close(); err != nil {
return err
}
}
f.views = make(map[string]*View)
f.viewMap = make(map[string]*View)
return nil
}
@ -482,8 +482,8 @@ func (f *Field) deleteBSIGroupAndView(name string) error {
// Remove views.
viewName := viewBSIGroupPrefix + name
if view := f.views[viewName]; view != nil {
delete(f.views, viewName)
if view := f.viewMap[viewName]; view != nil {
delete(f.viewMap, viewName)
if err := view.close(); err != nil {
return errors.Wrap(err, "closing view")
@ -540,22 +540,22 @@ func (f *Field) ViewPath(name string) string {
return filepath.Join(f.path, "views", name)
}
// View returns a view in the field by name.
func (f *Field) View(name string) *View {
// view returns a view in the field by name.
func (f *Field) view(name string) *View {
f.mu.RLock()
defer f.mu.RUnlock()
return f.view(name)
return f.unprotectedView(name)
}
func (f *Field) view(name string) *View { return f.views[name] }
func (f *Field) unprotectedView(name string) *View { return f.viewMap[name] }
// Views returns a list of all views in the field.
func (f *Field) Views() []*View {
// views returns a list of all views in the field.
func (f *Field) views() []*View {
f.mu.RLock()
defer f.mu.RUnlock()
other := make([]*View, 0, len(f.views))
for _, view := range f.views {
other := make([]*View, 0, len(f.viewMap))
for _, view := range f.viewMap {
other = append(other, view)
}
return other
@ -566,8 +566,8 @@ func (f *Field) viewNames() []string {
f.mu.Lock()
defer f.mu.Unlock()
other := make([]string, 0, len(f.views))
for viewName, _ := range f.views {
other := make([]string, 0, len(f.viewMap))
for viewName, _ := range f.viewMap {
other = append(other, viewName)
}
return other
@ -575,14 +575,14 @@ func (f *Field) viewNames() []string {
// RecalculateCaches recalculates caches on every view in the field.
func (f *Field) RecalculateCaches() {
for _, view := range f.Views() {
for _, view := range f.views() {
view.recalculateCaches()
}
}
// CreateViewIfNotExists returns the named view, creating it if necessary.
// createViewIfNotExists returns the named view, creating it if necessary.
// Additionally, a CreateViewMessage is sent to the cluster.
func (f *Field) CreateViewIfNotExists(name string) (*View, error) {
func (f *Field) createViewIfNotExists(name string) (*View, error) {
view, created, err := f.createViewIfNotExistsBase(name)
if err != nil {
return nil, err
@ -610,7 +610,7 @@ func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
if view := f.views[name]; view != nil {
if view := f.viewMap[name]; view != nil {
return view, false, nil
}
view := f.newView(f.ViewPath(name), name)
@ -619,7 +619,7 @@ func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) {
return nil, false, errors.Wrap(err, "opening view")
}
view.RowAttrStore = f.rowAttrStore
f.views[view.name] = view
f.viewMap[view.name] = view
return view, true, nil
}
@ -634,9 +634,9 @@ func (f *Field) newView(path, name string) *View {
return view
}
// DeleteView removes the view from the field.
func (f *Field) DeleteView(name string) error {
view := f.views[name]
// deleteView removes the view from the field.
func (f *Field) deleteView(name string) error {
view := f.viewMap[name]
if view == nil {
return ErrInvalidView
}
@ -651,7 +651,7 @@ func (f *Field) DeleteView(name string) error {
return errors.Wrap(err, "deleting directory")
}
delete(f.views, name)
delete(f.viewMap, name)
return nil
}
@ -661,7 +661,7 @@ func (f *Field) Row(rowID uint64) (*Row, error) {
if f.Type() != FieldTypeSet {
return nil, errors.Errorf("row method unsupported for field type: %s", f.Type())
}
view := f.View(ViewStandard)
view := f.view(ViewStandard)
if view == nil {
return nil, ErrInvalidView
}
@ -671,7 +671,7 @@ func (f *Field) Row(rowID uint64) (*Row, error) {
// ViewRow returns a row for a view and shard.
// TODO: unexport this with views (it's only used in tests).
func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) {
view := f.View(viewName)
view := f.view(viewName)
if view == nil {
return nil, ErrInvalidView
}
@ -683,7 +683,7 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err
viewName := ViewStandard
// Retrieve view. Exit if it doesn't exist.
view, err := f.CreateViewIfNotExists(viewName)
view, err := f.createViewIfNotExists(viewName)
if err != nil {
return changed, errors.Wrap(err, "creating view")
}
@ -702,7 +702,7 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err
// If a timestamp is specified then set bits across all views for the quantum.
for _, subname := range viewsByTime(viewName, *t, f.TimeQuantum()) {
view, err := f.CreateViewIfNotExists(subname)
view, err := f.createViewIfNotExists(subname)
if err != nil {
return changed, errors.Wrapf(err, "creating view %s", subname)
}
@ -722,7 +722,7 @@ func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) {
viewName := ViewStandard
// Retrieve view. Exit if it doesn't exist.
view, present := f.views[viewName]
view, present := f.viewMap[viewName]
if !present {
return changed, errors.Wrap(err, "clearing missing view")
@ -734,7 +734,7 @@ func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) {
} else if v {
changed = v
}
if len(f.views) == 1 { // assuming no time views
if len(f.viewMap) == 1 { // assuming no time views
return changed, nil
}
lastViewNameSize := 0
@ -774,11 +774,11 @@ func groupCompare(a, b string, offset int) (lt, eq bool) {
}
func (f *Field) allTimeViewsSortedByQuantum() (me []*View) {
me = make([]*View, len(f.views), len(f.views))
me = make([]*View, len(f.viewMap), len(f.viewMap))
prefix := ViewStandard + "_"
offset := len(ViewStandard) + 1
i := 0
for _, v := range f.views {
for _, v := range f.viewMap {
if len(v.name) > offset && strings.Compare(v.name[:offset], prefix) == 0 { // skip non-time views
me[i] = v
i++
@ -811,7 +811,7 @@ func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) {
}
// Fetch target view.
view := f.View(viewBSIGroupPrefix + f.name)
view := f.view(viewBSIGroupPrefix + f.name)
if view == nil {
return 0, false, nil
}
@ -838,7 +838,7 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error)
}
// Fetch target view.
view, err := f.CreateViewIfNotExists(viewBSIGroupPrefix + f.name)
view, err := f.createViewIfNotExists(viewBSIGroupPrefix + f.name)
if err != nil {
return false, errors.Wrap(err, "creating view")
}
@ -857,7 +857,7 @@ func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) {
return 0, 0, ErrBSIGroupNotFound
}
view := f.View(viewBSIGroupPrefix + name)
view := f.view(viewBSIGroupPrefix + name)
if view == nil {
return 0, 0, nil
}
@ -877,7 +877,7 @@ func (f *Field) Min(filter *Row, name string) (min, count int64, err error) {
return 0, 0, ErrBSIGroupNotFound
}
view := f.View(viewBSIGroupPrefix + name)
view := f.view(viewBSIGroupPrefix + name)
if view == nil {
return 0, 0, nil
}
@ -897,7 +897,7 @@ func (f *Field) Max(filter *Row, name string) (max, count int64, err error) {
return 0, 0, ErrBSIGroupNotFound
}
view := f.View(viewBSIGroupPrefix + name)
view := f.view(viewBSIGroupPrefix + name)
if view == nil {
return 0, 0, nil
}
@ -919,7 +919,7 @@ func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error)
}
// Retrieve bsiGroup's view.
view := f.View(viewBSIGroupPrefix + name)
view := f.view(viewBSIGroupPrefix + name)
if view == nil {
return nil, nil
}
@ -942,7 +942,7 @@ func (f *Field) RangeBetween(name string, predicateMin, predicateMax int64) (*Ro
}
// Retrieve bsiGroup's view.
view := f.View(viewBSIGroupPrefix + name)
view := f.view(viewBSIGroupPrefix + name)
if view == nil {
return nil, nil
}
@ -994,7 +994,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
// Import into each fragment.
for key, data := range dataByFragment {
view, err := f.CreateViewIfNotExists(key.View)
view, err := f.createViewIfNotExists(key.View)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -1046,7 +1046,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error {
// The view must already exist (i.e. we can't create it)
// because we need to know bitDepth (based on min/max value).
view, err := f.CreateViewIfNotExists(key.View)
view, err := f.createViewIfNotExists(key.View)
if err != nil {
return errors.Wrap(err, "creating view")
}

132
field_internal_test.go
View file

@ -15,6 +15,8 @@
package pilosa
import (
"io/ioutil"
"os"
"reflect"
"testing"
@ -147,3 +149,133 @@ func TestBSIGroup_BaseValue(t *testing.T) {
}
})
}
// Ensure field can open and retrieve a view.
func TestField_DeleteView(t *testing.T) {
f := MustOpenField(FieldOptions{})
defer f.Close()
viewName := ViewStandard + "_v"
// Create view.
view, err := f.createViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
err = f.deleteView(viewName)
if err != nil {
t.Fatal(err)
}
if f.view(viewName) != nil {
t.Fatal("view still exists in field")
}
// Recreate view with same name, verify that the old view was not reused.
view2, err := f.createViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == view2 {
t.Fatal("failed to create new view")
}
}
// TestField represents a test wrapper for Field.
type TestField struct {
*Field
}
// NewTestField returns a new instance of TestField d/0.
func NewTestField(options FieldOptions) *TestField {
path, err := ioutil.TempDir("", "pilosa-field-")
if err != nil {
panic(err)
}
field, err := NewField(path, "i", "f", options)
if err != nil {
panic(err)
}
return &TestField{Field: field}
}
// MustOpenField returns a new, opened field at a temporary path. Panic on error.
func MustOpenField(options FieldOptions) *TestField {
f := NewTestField(options)
if err := f.Open(); err != nil {
panic(err)
}
return f
}
// Close closes the field and removes the underlying data.
func (f *TestField) Close() error {
defer os.RemoveAll(f.Path())
return f.Field.Close()
}
// Reopen closes the index and reopens it.
func (f *TestField) Reopen() error {
var err error
if err := f.Field.Close(); err != nil {
return err
}
path, index, name := f.Path(), f.Index(), f.Name()
f.Field, err = NewField(path, index, name, FieldOptions{})
if err != nil {
return err
}
if err := f.Open(); err != nil {
return err
}
return nil
}
// Ensure field can open and retrieve a view.
func TestField_CreateViewIfNotExists(t *testing.T) {
f := MustOpenField(FieldOptions{})
defer f.Close()
// Create view.
view, err := f.createViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
// Retrieve existing view.
view2, err := f.createViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view != view2 {
t.Fatal("view mismatch")
}
if view != f.view("v") {
t.Fatal("view mismatch")
}
}
func TestField_SetTimeQuantum(t *testing.T) {
f := MustOpenField(FieldOptions{Type: FieldTypeTime})
defer f.Close()
// Set & retrieve time quantum.
if err := f.SetTimeQuantum(TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
}
// Reload field and verify that it is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum (reopen): %s", q)
}
}

79
field_test.go
View file

@ -22,52 +22,6 @@ import (
"github.com/pilosa/pilosa/test"
)
// Ensure field can open and retrieve a view.
func TestField_CreateViewIfNotExists(t *testing.T) {
f := test.MustOpenField(pilosa.FieldOptions{})
defer f.Close()
// Create view.
view, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
// Retrieve existing view.
view2, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view != view2 {
t.Fatal("view mismatch")
}
if view != f.View("v") {
t.Fatal("view mismatch")
}
}
// Ensure field can set its time quantum.
func TestField_SetTimeQuantum(t *testing.T) {
f := test.MustOpenField(pilosa.FieldOptions{Type: pilosa.FieldTypeTime})
defer f.Close()
// Set & retrieve time quantum.
if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
}
// Reload field and verify that it is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum (reopen): %s", q)
}
}
// Ensure a field can set & read a bsiGroup value.
func TestField_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
@ -249,36 +203,3 @@ func TestField_NameValidation(t *testing.T) {
}
}
}
// Ensure field can open and retrieve a view.
func TestField_DeleteView(t *testing.T) {
f := test.MustOpenField(pilosa.FieldOptions{})
defer f.Close()
viewName := pilosa.ViewStandard + "_v"
// Create view.
view, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
err = f.DeleteView(viewName)
if err != nil {
t.Fatal(err)
}
if f.View(viewName) != nil {
t.Fatal("view still exists in field")
}
// Recreate view with same name, verify that the old view was not reused.
view2, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == view2 {
t.Fatal("failed to create new view")
}
}

6
fragment_internal_test.go
View file

@ -756,7 +756,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
}
// Create view.
view, err := field.CreateViewIfNotExists(ViewStandard)
view, err := field.createViewIfNotExists(ViewStandard)
if err != nil {
t.Fatal(err)
}
@ -922,7 +922,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
}
// Create view.
view, err := field.CreateViewIfNotExists(ViewStandard)
view, err := field.createViewIfNotExists(ViewStandard)
if err != nil {
t.Fatal(err)
}
@ -953,7 +953,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
}
// Re-fetch fragment.
f = index.Field("f").View(ViewStandard).Fragment(0)
f = index.Field("f").view(ViewStandard).Fragment(0)
// Re-verify correct cache type and size.
if cache, ok := f.cache.(*RankCache); !ok {

12
holder.go
View file

@ -216,7 +216,7 @@ func (h *Holder) Schema() []*IndexInfo {
di := &IndexInfo{Name: index.Name()}
for _, field := range index.Fields() {
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
for _, view := range field.Views() {
for _, view := range field.views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
}
sort.Sort(viewInfoSlice(fi.Views))
@ -247,7 +247,7 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error {
}
// Create views that don't exist.
for _, v := range f.Views {
_, err := field.CreateViewIfNotExists(v)
_, err := field.createViewIfNotExists(v)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -408,7 +408,7 @@ func (h *Holder) View(index, field, name string) *View {
if f == nil {
return nil
}
return f.View(name)
return f.view(name)
}
// Fragment returns the fragment for an index, field & shard.
@ -439,7 +439,7 @@ func (h *Holder) monitorCacheFlush() {
func (h *Holder) flushCaches() {
for _, index := range h.Indexes() {
for _, field := range index.Fields() {
for _, view := range field.Views() {
for _, view := range field.views() {
for _, fragment := range view.allFragments() {
select {
case <-h.closing:
@ -748,7 +748,7 @@ func (s *HolderSyncer) syncFragment(index, field, view string, shard uint64) err
}
// Ensure view exists locally.
v, err := f.CreateViewIfNotExists(view)
v, err := f.createViewIfNotExists(view)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -808,7 +808,7 @@ func (c *HolderCleaner) CleanHolder() error {
// Get the fragments registered in memory.
for _, field := range index.Fields() {
for _, view := range field.Views() {
for _, view := range field.views() {
for _, fragment := range view.allFragments() {
fragShard := fragment.shard
// Ignore fragments that should be present.

139
holder_internal_test.go Normal file
View file

@ -0,0 +1,139 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
)
type tHolder struct {
*Holder
}
// Close closes the holder and removes all underlying data.
func (h *tHolder) Close() error {
defer os.RemoveAll(h.Path)
return h.Holder.Close()
}
// Reopen instantiates and opens a new holder.
// Note that the holder must be Closed first.
func (h *tHolder) Reopen() error {
path, logger := h.Path, h.Holder.Logger
h.Holder = NewHolder()
h.Holder.Path = path
h.Holder.Logger = logger
if err := h.Holder.Open(); err != nil {
return err
}
return nil
}
func newHolder() *tHolder {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
h := &tHolder{Holder: NewHolder()}
h.Path = path
return h
}
func TestHolder_Optn(t *testing.T) {
t.Run("ErrViewPermission", func(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
h := newHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := field.createViewIfNotExists(ViewStandard); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil {
t.Fatal(err)
}
defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777)
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrViewFragmentsMkdir", func(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
h := newHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := field.createViewIfNotExists(ViewStandard); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil {
t.Fatal(err)
}
defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777)
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFragmentCachePermission", func(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
h := newHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil {
t.Fatal(err)
} else if view, err := field.createViewIfNotExists(ViewStandard); err != nil {
t.Fatal(err)
} else if _, err := field.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
} else if err := view.Fragment(0).FlushCache(); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil {
t.Fatal(err)
}
defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666)
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("unexpected error: %s", err)
}
})
}

77
holder_test.go
View file

@ -148,55 +148,6 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrViewPermission", func(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil {
t.Fatal(err)
}
defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777)
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrViewFragmentsMkdir", func(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil {
t.Fatal(err)
}
defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777)
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
@ -242,34 +193,6 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFragmentCachePermission", func(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if _, err := field.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
} else if err := view.Fragment(0).FlushCache(); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil {
t.Fatal(err)
}
defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666)
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("unexpected error: %s", err)
}
})
}
func TestHolder_HasData(t *testing.T) {

2
server.go
View file

@ -479,7 +479,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if f == nil {
return fmt.Errorf("Local Field not found: %s", obj.Field)
}
err := f.DeleteView(obj.View)
err := f.deleteView(obj.View)
if err != nil {
return err
}

18
test/holder.go
View file

@ -89,24 +89,6 @@ func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field {
return f
}
// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error.
func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, shard uint64) *Fragment {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
v, err := f.CreateViewIfNotExists(view)
if err != nil {
panic(err)
}
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
panic(err)
}
return &Fragment{Fragment: frag}
}
// Row returns a Row for a given field.
func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})

2
utils_internal_test.go
View file

@ -373,7 +373,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
if destFragment == nil {
// Create fragment on destination if it doesn't exist.
f := destCluster.holder.Field(src.Index, src.Field)
v := f.View(src.View)
v := f.view(src.View)
var err error
destFragment, err = v.CreateFragmentIfNotExists(src.Shard)
if err != nil {