From bfc0e56dc2edc0e5428ea814dce024746030c654 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 2 Mar 2017 15:17:21 -0600 Subject: [PATCH] move BitmapCache into cache.go. adjust some variable names for clarity --- bitmap.go | 2 +- cache.go | 35 +++++++++++++++++++++++++----- fragment.go | 62 ++++++++++++++++------------------------------------- 3 files changed, 50 insertions(+), 49 deletions(-) diff --git a/bitmap.go b/bitmap.go index d30a9192a..651950e45 100644 --- a/bitmap.go +++ b/bitmap.go @@ -17,7 +17,7 @@ type Bitmap struct { // Attributes associated with the bitmap. Attrs map[string]interface{} - cacheoveride uint64 + adjustedCount uint64 } // NewBitmap returns a new instance of Bitmap. diff --git a/cache.go b/cache.go index 4345c93fd..184310361 100644 --- a/cache.go +++ b/cache.go @@ -176,16 +176,17 @@ func (c *RankCache) Invalidate() { } func (c *RankCache) invalidate() { + // Don't invalidate more than once every X seconds. + // TODO: consider making this configurable. if time.Now().Sub(c.updateTime).Seconds() < 10 { return } - //fmt.Println("RankCache Update") // Convert cache to a sorted list. rankings := make([]BitmapPair, 0, len(c.entries)) - for id, n := range c.entries { + for id, cnt := range c.entries { rankings = append(rankings, BitmapPair{ ID: id, - Count: n, + Count: cnt, }) } sort.Sort(BitmapPairs(rankings)) @@ -200,10 +201,11 @@ func (c *RankCache) invalidate() { // Reset counters. c.updateTime, c.updateN = time.Now(), 0 + // If size is larger than the threshold then trim it. if len(c.entries) > c.ThresholdLength { - for id, n := range c.entries { - if n <= c.ThresholdValue { + for id, cnt := range c.entries { + if cnt <= c.ThresholdValue { delete(c.entries, id) } } @@ -378,3 +380,26 @@ func (p uint64Slice) merge(other []uint64) []uint64 { return ret } + +// BitmapCache provides an interface for caching full bitmaps. +type BitmapCache interface { + Fetch(id uint64) (*Bitmap, bool) + Add(id uint64, b *Bitmap) +} + +// SimpleCache implements BitmapCache +// it is meant to be a short-lived cache for cases where writes are continuing to access +// the same bit within a short time frame (i.e. good for write-heavy loads) +// A read-heavy use case would cause the cache to get bigger, potentially causing the +// node to run out of memory. +type SimpleCache struct { + cache map[uint64]*Bitmap +} + +func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) { + return s.cache[id] +} + +func (s *SimpleCache) Add(id uint64, b *Bitmap) { + s.cache[id] = b +} diff --git a/fragment.go b/fragment.go index e203dd711..8f8698192 100644 --- a/fragment.go +++ b/fragment.go @@ -53,28 +53,6 @@ const ( DefaultFragmentMaxOpN = 2000 ) -// BitmapCacher implements SimpleCache -// it is meant to be a short-lived cache for cases where writes are continuing to access -// the same bit withing a short time frame (i.e. good for write-heavy loads) -// A read-heavy use case would cause the cache to get bigger, potentially causing the -// node to run out of memory. -type BitmapCacher interface { - Fetch(id uint64) (*Bitmap, bool) - Add(id uint64, b *Bitmap) -} - -type SimpleCache struct { - cache map[uint64]*Bitmap -} - -func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) { - m, ok := s.cache[id] - return m, ok -} -func (s *SimpleCache) Add(id uint64, p *Bitmap) { - s.cache[id] = p -} - // Fragment represents the intersection of a frame and slice in a database. type Fragment struct { mu sync.Mutex @@ -91,9 +69,12 @@ type Fragment struct { storageData []byte opN int // number of ops since snapshot - // Bitmap cache. + // Cache for bitmap counts. cache Cache + // Cache containing full bitmaps (not just counts). + bitmapCache BitmapCache + // Cached checksums for each block. checksums map[int][]byte @@ -109,8 +90,7 @@ type Fragment struct { // This is set by the parent frame unless overridden for testing. BitmapAttrStore *AttrStore - stats StatsClient - bitmapCache BitmapCacher + stats StatsClient } // NewFragment returns a new instance of Fragment. @@ -351,10 +331,10 @@ func (f *Fragment) bitmap(bitmapID uint64, updateCache bool) *Bitmap { slice: f.slice, writable: false, }}, - cacheoveride: 0, + adjustedCount: 0, } bm.InvalidateCount() - bm.cacheoveride = bm.Count() + bm.adjustedCount = bm.Count() if updateCache { // Update cache. @@ -382,7 +362,6 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) } // Write to storage. - if changed, err = f.storage.Add(pos); err != nil { return false, err } @@ -400,18 +379,17 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) return false, err } - // Update the cache. + // If adjustedCount is set, then apply that value to bitmap.n instead. bm := f.bitmap(bitmapID, true) - if bm.cacheoveride > 0 { - bm.SetCount(profileID, bm.cacheoveride) - bm.cacheoveride = 0 + if bm.adjustedCount > 0 { + bm.SetCount(profileID, bm.adjustedCount) + bm.adjustedCount = 0 } else { bm.IncrementCount(profileID) + f.cache.Add(bitmapID, bm.Count()) } bm.SetBit(profileID) - f.cache.Add(bitmapID, bm.Count()) - f.stats.Count("setN", 1) return changed, nil @@ -451,15 +429,16 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (bool, error) { return false, err } - // Update the cache. - bm := f.bitmap(bitmapID, false) - if bm.cacheoveride > 0 { - bm.SetCount(profileID, bm.cacheoveride) - bm.cacheoveride = 0 + // If adjustedCount is set, then apply that value to bitmap.n instead. + bm := f.bitmap(bitmapID, true) + if bm.adjustedCount > 0 { + bm.SetCount(profileID, bm.adjustedCount) + bm.adjustedCount = 0 } else { bm.DecrementCount(profileID) + f.cache.Add(bitmapID, bm.Count()) } - f.cache.Add(bitmapID, bm.Count()) + bm.ClearBit(profileID) f.stats.Count("clearN", 1) @@ -546,7 +525,6 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { if count == 0 { continue } - //results = append(results, Pair{Key: bitmapID, Count: count}) heap.Push(results, Pair{Key: bitmapID, Count: count}) // If we reach the requested number of pairs and we are not computing @@ -556,7 +534,6 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { if opt.Src == nil { break } - // sort.Sort(Pairs(results)) } continue } @@ -622,7 +599,6 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { } } sort.Sort(BitmapPairs(pairs)) - //debugDumpPairs(pairs) return pairs }