diff --git a/fragment.go b/fragment.go index 063b7e17c..fe04682dc 100644 --- a/fragment.go +++ b/fragment.go @@ -70,6 +70,7 @@ type Fragment struct { // Cache for bitmap counts. cacheType string // passed in by frame cache Cache + cacheSize int // Cache containing full bitmaps (not just counts). bitmapCache BitmapCache @@ -93,7 +94,7 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, db, frame, view string, slice uint64) *Fragment { +func NewFragment(path, db, frame, view string, slice uint64, cacheSize int) *Fragment { return &Fragment{ path: path, db: db, @@ -101,6 +102,7 @@ func NewFragment(path, db, frame, view string, slice uint64) *Fragment { view: view, slice: slice, cacheType: DefaultCacheType, + cacheSize: cacheSize, LogOutput: ioutil.Discard, MaxOpN: DefaultFragmentMaxOpN, @@ -217,21 +219,43 @@ func (f *Fragment) openStorage() error { } +// // It will be the one and only identifier after a package specifier. +// var testNameRegexp = regexp.MustCompile(`\.(Test[\p{L}_\p{N}]*)$`) + +// // Returns the name of the test function from the call stack. See +// // http://stackoverflow.com/q/35535635/149482 for another method. +// func GetTestName() string { +// pc := make([]uintptr, 32) +// n := runtime.Callers(0, pc) +// for i := 0; i < n; i++ { +// name := runtime.FuncForPC(pc[i]).Name() +// ms := testNameRegexp.FindStringSubmatch(name) +// if ms == nil { +// continue +// } +// return ms[1] +// } +// panic("test name could not be recovered") +// } + // openCache initializes the cache from bitmap ids persisted to disk. func (f *Fragment) openCache() error { // Determine cache type from frame name. switch f.cacheType { case CacheTypeRanked: c := NewRankCache() - c.ThresholdLength = 50000 - c.ThresholdIndex = 45000 + c.ThresholdLength = f.cacheSize + c.ThresholdIndex = f.cacheSize - 500 f.cache = c case CacheTypeLRU: - f.cache = NewLRUCache(50000) + f.cache = NewLRUCache(f.cacheSize) default: return ErrInvalidCacheType } + // fmt.Println(GetTestName()) + // fmt.Printf("CACHE SIZE: %d\n", f.cacheSize) + // Read cache data from disk. path := f.CachePath() buf, err := ioutil.ReadFile(path) diff --git a/fragment_test.go b/fragment_test.go index b915bd449..ca6ffa3dc 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -277,6 +277,44 @@ func TestFragment_TopN_BitmapIDs(t *testing.T) { } } +// Ensure the fragment cache limit works +func TestFragment_TopN_CacheSize(t *testing.T) { + slice := uint64(0) + cacheLimit := 3 + file, err := ioutil.TempFile("", "pilosa-fragment-") + if err != nil { + panic(err) + } + file.Close() + + f := &Fragment{ + Fragment: pilosa.NewFragment(file.Name(), "d", "f", slice, cacheLimit), + BitmapAttrStore: MustOpenAttrStore(), + } + f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore + if err := f.Open(); err != nil { + panic(err) + } + defer f.Close() + + // Set bits on various bitmaps. + f.MustSetBits(100, 1, 2, 3) + f.MustSetBits(101, 4, 5, 6, 7) + f.MustSetBits(102, 8, 9, 10, 11, 12) + f.MustSetBits(103, 8, 9, 10, 11, 12, 13) + f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14) + f.MustSetBits(105, 10, 11) + + // Retrieve top bitmaps. + if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { + t.Fatal(err) + } else if len(pairs) != cacheLimit { + t.Fatalf("TopN count cannot exceed cache size: %d", len(pairs)) + } else if pairs[0] != (pilosa.Pair{Key: 104, Count: 7}) { + t.Fatalf("unexpected pair(0): %v", pairs[0]) + } +} + // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) @@ -526,7 +564,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0) + f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0, pilosa.DefaultFrameCache) if err := f.Open(); err != nil { b.Fatal(err) } @@ -587,7 +625,7 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), + Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice, pilosa.DefaultFrameCache), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -618,7 +656,7 @@ func (f *Fragment) Reopen() error { return err } - f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice()) + f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice(), pilosa.DefaultFrameCache) f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore if err := f.Open(); err != nil { return err diff --git a/frame.go b/frame.go index c70a07b68..be949b19d 100644 --- a/frame.go +++ b/frame.go @@ -20,6 +20,9 @@ const ( DefaultRowLabel = "id" DefaultCacheType = CacheTypeLRU DefaultInverseEnabled = false + + // Default ranked frame cache + DefaultFrameCache = 50000 ) // Frame represents a container for views. @@ -42,6 +45,9 @@ type Frame struct { cacheType string inverseEnabled bool + // Cache size for ranked frames + rankedCacheSize int + LogOutput io.Writer } @@ -62,9 +68,10 @@ func NewFrame(path, db, name string) (*Frame, error) { stats: NopStatsClient, - rowLabel: DefaultRowLabel, - cacheType: DefaultCacheType, - inverseEnabled: DefaultInverseEnabled, + rowLabel: DefaultRowLabel, + cacheType: DefaultCacheType, + inverseEnabled: DefaultInverseEnabled, + rankedCacheSize: DefaultFrameCache, LogOutput: ioutil.Discard, }, nil @@ -149,13 +156,42 @@ func (f *Frame) InverseEnabled() bool { return f.inverseEnabled } +// SetRankedCacheSize sets the cache size for ranked fames. Persists to meta file on update. +// defaults to DefaultFrameCache 50000 +func (f *Frame) SetRankedCacheSize(v int) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Ignore if no change occurred. + if v == 0 || f.rankedCacheSize == v { + return nil + } + + // Persist meta data to disk on change. + f.rankedCacheSize = v + if err := f.saveMeta(); err != nil { + return err + } + + return nil +} + +// RankedCacheSize returns the ranked frame cache size. +func (f *Frame) RankedCacheSize() int { + f.mu.Lock() + v := f.rankedCacheSize + f.mu.Unlock() + return v +} + // Options returns all options for this frame. func (f *Frame) Options() FrameOptions { f.mu.Lock() opt := FrameOptions{ RowLabel: f.rowLabel, - CacheType: f.cacheType, InverseEnabled: f.inverseEnabled, + CacheType: f.cacheType, + CacheSize: f.rankedCacheSize, } f.mu.Unlock() return opt @@ -559,8 +595,9 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { RowLabel string `json:"rowLabel,omitempty"` - CacheType string `json:"cacheType,omitempty"` InverseEnabled bool `json:"inverseEnabled,omitempty"` + CacheType string `json:"cacheType,omitempty"` + CacheSize int `json:"cacheSize,omitempty"` } // importBitSet represents slices of row and column ids. diff --git a/view.go b/view.go index 54dda6f56..0f554de92 100644 --- a/view.go +++ b/view.go @@ -30,6 +30,8 @@ type View struct { frame string name string + cacheSize int + // Fragments by slice. cacheType string // passed in by frame fragments map[uint64]*Fragment @@ -41,12 +43,13 @@ type View struct { } // NewView returns a new instance of View. -func NewView(path, db, frame, name string) *View { +func NewView(path, db, frame, name string, cacheSize int) *View { return &View{ - path: path, - db: db, - frame: frame, - name: name, + path: path, + db: db, + frame: frame, + name: name, + cacheSize: cacheSize, cacheType: DefaultCacheType, fragments: make(map[uint64]*Fragment), @@ -213,7 +216,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } func (v *View) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, v.db, v.frame, v.name, slice) + frag := NewFragment(path, v.db, v.frame, v.name, slice, v.cacheSize) frag.cacheType = v.cacheType frag.LogOutput = v.LogOutput frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) diff --git a/view_test.go b/view_test.go index 87b3e5826..d10c90936 100644 --- a/view_test.go +++ b/view_test.go @@ -22,7 +22,7 @@ func NewView(db, frame, name string) *View { file.Close() v := &View{ - View: pilosa.NewView(file.Name(), db, frame, name), + View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize), BitmapAttrStore: MustOpenAttrStore(), } v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore @@ -52,7 +52,7 @@ func (v *View) Reopen() error { return err } - v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name()) + v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore if err := v.Open(); err != nil { return err