scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make sure the TopN test is actually using a RankCache supported frame.

Browse source 
Implement cache.Recalculate to bypass the 10-second delay built into cache.Invalidate().
This commit is contained in:
Travis 2017-03-10 11:07:18 -06:00
parent 988e602033
commit 2b94be965e
3 changed files with 51 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
cache.go
View file

@ -25,6 +25,9 @@ type Cache interface {
// Updates the cache, if necessary.
Invalidate()
// Rebuilds the cache
Recalculate()
// Returns an ordered list of the top ranked bitmaps.
Top() []BitmapPair
}
@ -68,6 +71,9 @@ func (c *LRUCache) Len() int { return c.cache.Len() }
// Invalidate is a no-op.
func (c *LRUCache) Invalidate() {}
// Recalculate is a no-op.
func (c *LRUCache) Recalculate() {}
// BitmapIDs returns a list of all bitmap IDs in the cache.
func (c *LRUCache) BitmapIDs() []uint64 {
a := make([]uint64, 0, len(c.counts))
@ -168,19 +174,29 @@ func (c *RankCache) BitmapIDs() []uint64 {
return a
}
// update reorders the entries by rank.
// Invalidate recalculates the the entries by rank.
func (c *RankCache) Invalidate() {
c.mu.Lock()
defer c.mu.Unlock()
c.invalidate()
}
func (c *RankCache) Recalculate() {
c.mu.Lock()
defer c.mu.Unlock()
c.recalculate()
}
func (c *RankCache) invalidate() {
// Don't invalidate more than once every X seconds.
// TODO: consider making this configurable.
if time.Now().Sub(c.updateTime).Seconds() < 10 {
return
}
c.recalculate()
}
func (c *RankCache) recalculate() {
// Convert cache to a sorted list.
rankings := make([]BitmapPair, 0, len(c.entries))
for id, cnt := range c.entries {

45
client_test.go
View file

@ -61,26 +61,33 @@ func TestClient_MultiNode(t *testing.T) {
}
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
idx[0].MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(99, 1, 2, 3, 4)
idx[0].MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
idx[0].MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(98, 1, 2, 3, 4, 5, 6)
idx[0].MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(1, 4)
idx[0].MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(22, 1, 2, 3, 4, 5)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(99, 1, 2, 3, 4)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(98, 1, 2, 3, 4, 5, 6)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(1, 4)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(22, 1, 2, 3, 4, 5)
idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(100, (SliceWidth*10)+10)
idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12)
idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15)
idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4)
idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5)
idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(100, (SliceWidth*10)+10)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10)
idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14)
idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13)
idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(21, (SliceWidth*6)+10)
idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(100, (SliceWidth*6)+10)
idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11)
idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(21, (SliceWidth*6)+10)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(100, (SliceWidth*6)+10)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
// Rebuild the RankCache.
// We have to do this to avoid the 10-second cache invalidation delay
// built into cache.Invalidate()
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).RecalculateCache()
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).RecalculateCache()
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).RecalculateCache()
// Connect to each node to compare results.
client := make([]*Client, 3)
@ -89,7 +96,7 @@ func TestClient_MultiNode(t *testing.T) {
client[2] = MustNewClient(s[0].Host())
topN := 4
q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN)
q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f.n", topN)
result, err := client[0].ExecuteQuery(context.Background(), "d", q, true)
if err != nil {

7
fragment.go
View file

@ -997,6 +997,13 @@ func (f *Fragment) snapshot() error {
return nil
}
// RecalculateCache rebuilds the cache regardless of invalidate time delay.
func (f *Fragment) RecalculateCache() {
f.mu.Lock()
f.cache.Recalculate()
f.mu.Unlock()
}
// FlushCache writes the cache data to disk.
func (f *Fragment) FlushCache() error {
f.mu.Lock()