fragment cache size is set on frame creation with a default value of 50,000

fragment.go

@@ -70,6 +70,7 @@ type Fragment struct {
 	// Cache for bitmap counts.
 	cacheType string // passed in by frame
 	cache     Cache
+	cacheSize int
 
 	// Cache containing full bitmaps (not just counts).
 	bitmapCache BitmapCache
@@ -93,7 +94,7 @@ type Fragment struct {
 }
 
 // NewFragment returns a new instance of Fragment.
-func NewFragment(path, db, frame, view string, slice uint64) *Fragment {
+func NewFragment(path, db, frame, view string, slice uint64, cacheSize int) *Fragment {
 	return &Fragment{
 		path:      path,
 		db:        db,
@@ -101,6 +102,7 @@ func NewFragment(path, db, frame, view string, slice uint64) *Fragment {
 		view:      view,
 		slice:     slice,
 		cacheType: DefaultCacheType,
+		cacheSize: cacheSize,
 
 		LogOutput: ioutil.Discard,
 		MaxOpN:    DefaultFragmentMaxOpN,
@@ -217,21 +219,43 @@ func (f *Fragment) openStorage() error {
 
 }
 
+// // It will be the one and only identifier after a package specifier.
+// var testNameRegexp = regexp.MustCompile(`\.(Test[\p{L}_\p{N}]*)$`)
+
+// // Returns the name of the test function from the call stack. See
+// // http://stackoverflow.com/q/35535635/149482 for another method.
+// func GetTestName() string {
+// 	pc := make([]uintptr, 32)
+// 	n := runtime.Callers(0, pc)
+// 	for i := 0; i < n; i++ {
+// 		name := runtime.FuncForPC(pc[i]).Name()
+// 		ms := testNameRegexp.FindStringSubmatch(name)
+// 		if ms == nil {
+// 			continue
+// 		}
+// 		return ms[1]
+// 	}
+// 	panic("test name could not be recovered")
+// }
+
 // openCache initializes the cache from bitmap ids persisted to disk.
 func (f *Fragment) openCache() error {
 	// Determine cache type from frame name.
 	switch f.cacheType {
 	case CacheTypeRanked:
 		c := NewRankCache()
-		c.ThresholdLength = 50000
-		c.ThresholdIndex = 45000
+		c.ThresholdLength = f.cacheSize
+		c.ThresholdIndex = f.cacheSize - 500
 		f.cache = c
 	case CacheTypeLRU:
-		f.cache = NewLRUCache(50000)
+		f.cache = NewLRUCache(f.cacheSize)
 	default:
 		return ErrInvalidCacheType
 	}
 
+	// fmt.Println(GetTestName())
+	// fmt.Printf("CACHE SIZE: %d\n", f.cacheSize)
+
 	// Read cache data from disk.
 	path := f.CachePath()
 	buf, err := ioutil.ReadFile(path)

fragment_test.go

@@ -277,6 +277,44 @@ func TestFragment_TopN_BitmapIDs(t *testing.T) {
 	}
 }
 
+// Ensure the fragment cache limit works
+func TestFragment_TopN_CacheSize(t *testing.T) {
+	slice := uint64(0)
+	cacheLimit := 3
+	file, err := ioutil.TempFile("", "pilosa-fragment-")
+	if err != nil {
+		panic(err)
+	}
+	file.Close()
+
+	f := &Fragment{
+		Fragment:        pilosa.NewFragment(file.Name(), "d", "f", slice, cacheLimit),
+		BitmapAttrStore: MustOpenAttrStore(),
+	}
+	f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore
+	if err := f.Open(); err != nil {
+		panic(err)
+	}
+	defer f.Close()
+
+	// Set bits on various bitmaps.
+	f.MustSetBits(100, 1, 2, 3)
+	f.MustSetBits(101, 4, 5, 6, 7)
+	f.MustSetBits(102, 8, 9, 10, 11, 12)
+	f.MustSetBits(103, 8, 9, 10, 11, 12, 13)
+	f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14)
+	f.MustSetBits(105, 10, 11)
+
+	// Retrieve top bitmaps.
+	if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil {
+		t.Fatal(err)
+	} else if len(pairs) != cacheLimit {
+		t.Fatalf("TopN count cannot exceed cache size: %d", len(pairs))
+	} else if pairs[0] != (pilosa.Pair{Key: 104, Count: 7}) {
+		t.Fatalf("unexpected pair(0): %v", pairs[0])
+	}
+}
+
 // Ensure fragment can return a checksum for its blocks.
 func TestFragment_Checksum(t *testing.T) {
 	f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
@@ -526,7 +564,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
 	}
 
 	// Open the fragment specified by the path.
-	f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0)
+	f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0, pilosa.DefaultFrameCache)
 	if err := f.Open(); err != nil {
 		b.Fatal(err)
 	}
@@ -587,7 +625,7 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment {
 	file.Close()
 
 	f := &Fragment{
-		Fragment:        pilosa.NewFragment(file.Name(), db, frame, view, slice),
+		Fragment:        pilosa.NewFragment(file.Name(), db, frame, view, slice, pilosa.DefaultFrameCache),
 		BitmapAttrStore: MustOpenAttrStore(),
 	}
 	f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore
@@ -618,7 +656,7 @@ func (f *Fragment) Reopen() error {
 		return err
 	}
 
-	f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice())
+	f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice(), pilosa.DefaultFrameCache)
 	f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore
 	if err := f.Open(); err != nil {
 		return err

frame.go

@@ -20,6 +20,9 @@ const (
 	DefaultRowLabel       = "id"
 	DefaultCacheType      = CacheTypeLRU
 	DefaultInverseEnabled = false
+
+	// Default ranked frame cache
+	DefaultFrameCache = 50000
 )
 
 // Frame represents a container for views.
@@ -42,6 +45,9 @@ type Frame struct {
 	cacheType      string
 	inverseEnabled bool
 
+	// Cache size for ranked frames
+	rankedCacheSize int
+
 	LogOutput io.Writer
 }
 
@@ -62,9 +68,10 @@ func NewFrame(path, db, name string) (*Frame, error) {
 
 		stats: NopStatsClient,
 
-		rowLabel:       DefaultRowLabel,
-		cacheType:      DefaultCacheType,
-		inverseEnabled: DefaultInverseEnabled,
+		rowLabel:        DefaultRowLabel,
+		cacheType:       DefaultCacheType,
+		inverseEnabled:  DefaultInverseEnabled,
+		rankedCacheSize: DefaultFrameCache,
 
 		LogOutput: ioutil.Discard,
 	}, nil
@@ -149,13 +156,42 @@ func (f *Frame) InverseEnabled() bool {
 	return f.inverseEnabled
 }
 
+// SetRankedCacheSize sets the cache size for ranked fames. Persists to meta file on update.
+// defaults to DefaultFrameCache 50000
+func (f *Frame) SetRankedCacheSize(v int) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	// Ignore if no change occurred.
+	if v == 0 || f.rankedCacheSize == v {
+		return nil
+	}
+
+	// Persist meta data to disk on change.
+	f.rankedCacheSize = v
+	if err := f.saveMeta(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RankedCacheSize returns the ranked frame cache size.
+func (f *Frame) RankedCacheSize() int {
+	f.mu.Lock()
+	v := f.rankedCacheSize
+	f.mu.Unlock()
+	return v
+}
+
 // Options returns all options for this frame.
 func (f *Frame) Options() FrameOptions {
 	f.mu.Lock()
 	opt := FrameOptions{
 		RowLabel:       f.rowLabel,
-		CacheType:      f.cacheType,
 		InverseEnabled: f.inverseEnabled,
+		CacheType:      f.cacheType,
+		CacheSize:      f.rankedCacheSize,
 	}
 	f.mu.Unlock()
 	return opt
@@ -559,8 +595,9 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
 // FrameOptions represents options to set when initializing a frame.
 type FrameOptions struct {
 	RowLabel       string `json:"rowLabel,omitempty"`
-	CacheType      string `json:"cacheType,omitempty"`
 	InverseEnabled bool   `json:"inverseEnabled,omitempty"`
+	CacheType      string `json:"cacheType,omitempty"`
+	CacheSize      int    `json:"cacheSize,omitempty"`
 }
 
 // importBitSet represents slices of row and column ids.

view.go

@@ -30,6 +30,8 @@ type View struct {
 	frame string
 	name  string
 
+	cacheSize int
+
 	// Fragments by slice.
 	cacheType string // passed in by frame
 	fragments map[uint64]*Fragment
@@ -41,12 +43,13 @@ type View struct {
 }
 
 // NewView returns a new instance of View.
-func NewView(path, db, frame, name string) *View {
+func NewView(path, db, frame, name string, cacheSize int) *View {
 	return &View{
-		path:  path,
-		db:    db,
-		frame: frame,
-		name:  name,
+		path:      path,
+		db:        db,
+		frame:     frame,
+		name:      name,
+		cacheSize: cacheSize,
 
 		cacheType: DefaultCacheType,
 		fragments: make(map[uint64]*Fragment),
@@ -213,7 +216,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
 }
 
 func (v *View) newFragment(path string, slice uint64) *Fragment {
-	frag := NewFragment(path, v.db, v.frame, v.name, slice)
+	frag := NewFragment(path, v.db, v.frame, v.name, slice, v.cacheSize)
 	frag.cacheType = v.cacheType
 	frag.LogOutput = v.LogOutput
 	frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice))

view_test.go

@@ -22,7 +22,7 @@ func NewView(db, frame, name string) *View {
 	file.Close()
 
 	v := &View{
-		View:            pilosa.NewView(file.Name(), db, frame, name),
+		View:            pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize),
 		BitmapAttrStore: MustOpenAttrStore(),
 	}
 	v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore
@@ -52,7 +52,7 @@ func (v *View) Reopen() error {
 		return err
 	}
 
-	v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name())
+	v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name(), pilosa.DefaultCacheSize)
 	v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore
 	if err := v.Open(); err != nil {
 		return err