From 2b94be965ee502e0fdf1bf6eb2e2e69ae764686d Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 10 Mar 2017 11:07:18 -0600 Subject: [PATCH] Make sure the TopN test is actually using a RankCache supported frame. Implement cache.Recalculate to bypass the 10-second delay built into cache.Invalidate(). --- cache.go | 20 ++++++++++++++++++-- client_test.go | 45 ++++++++++++++++++++++++++------------------- fragment.go | 7 +++++++ 3 files changed, 51 insertions(+), 21 deletions(-) diff --git a/cache.go b/cache.go index 7c2e96f2d..0185ce260 100644 --- a/cache.go +++ b/cache.go @@ -25,6 +25,9 @@ type Cache interface { // Updates the cache, if necessary. Invalidate() + // Rebuilds the cache + Recalculate() + // Returns an ordered list of the top ranked bitmaps. Top() []BitmapPair } @@ -68,6 +71,9 @@ func (c *LRUCache) Len() int { return c.cache.Len() } // Invalidate is a no-op. func (c *LRUCache) Invalidate() {} +// Recalculate is a no-op. +func (c *LRUCache) Recalculate() {} + // BitmapIDs returns a list of all bitmap IDs in the cache. func (c *LRUCache) BitmapIDs() []uint64 { a := make([]uint64, 0, len(c.counts)) @@ -168,19 +174,29 @@ func (c *RankCache) BitmapIDs() []uint64 { return a } -// update reorders the entries by rank. +// Invalidate recalculates the the entries by rank. func (c *RankCache) Invalidate() { c.mu.Lock() defer c.mu.Unlock() c.invalidate() - } + +func (c *RankCache) Recalculate() { + c.mu.Lock() + defer c.mu.Unlock() + c.recalculate() +} + func (c *RankCache) invalidate() { // Don't invalidate more than once every X seconds. // TODO: consider making this configurable. if time.Now().Sub(c.updateTime).Seconds() < 10 { return } + c.recalculate() +} + +func (c *RankCache) recalculate() { // Convert cache to a sorted list. rankings := make([]BitmapPair, 0, len(c.entries)) for id, cnt := range c.entries { diff --git a/client_test.go b/client_test.go index 7466d8639..15af73d8a 100644 --- a/client_test.go +++ b/client_test.go @@ -61,26 +61,33 @@ func TestClient_MultiNode(t *testing.T) { } // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. - idx[0].MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(99, 1, 2, 3, 4) - idx[0].MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - idx[0].MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(98, 1, 2, 3, 4, 5, 6) - idx[0].MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(1, 4) - idx[0].MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(22, 1, 2, 3, 4, 5) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(99, 1, 2, 3, 4) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(98, 1, 2, 3, 4, 5, 6) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(1, 4) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(22, 1, 2, 3, 4, 5) - idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(100, (SliceWidth*10)+10) - idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12) - idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15) - idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4) - idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5) - idx[1].MustCreateFragmentIfNotExists("d", "f", 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(100, (SliceWidth*10)+10) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14) - idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13) - idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(21, (SliceWidth*6)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(100, (SliceWidth*6)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) - idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11) - idx[2].MustCreateFragmentIfNotExists("d", "f", 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(21, (SliceWidth*6)+10) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(100, (SliceWidth*6)+10) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) + + // Rebuild the RankCache. + // We have to do this to avoid the 10-second cache invalidation delay + // built into cache.Invalidate() + idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).RecalculateCache() + idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).RecalculateCache() + idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).RecalculateCache() // Connect to each node to compare results. client := make([]*Client, 3) @@ -89,7 +96,7 @@ func TestClient_MultiNode(t *testing.T) { client[2] = MustNewClient(s[0].Host()) topN := 4 - q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN) + q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f.n", topN) result, err := client[0].ExecuteQuery(context.Background(), "d", q, true) if err != nil { diff --git a/fragment.go b/fragment.go index 4630a4ac4..9f7f155b9 100644 --- a/fragment.go +++ b/fragment.go @@ -997,6 +997,13 @@ func (f *Fragment) snapshot() error { return nil } +// RecalculateCache rebuilds the cache regardless of invalidate time delay. +func (f *Fragment) RecalculateCache() { + f.mu.Lock() + f.cache.Recalculate() + f.mu.Unlock() +} + // FlushCache writes the cache data to disk. func (f *Fragment) FlushCache() error { f.mu.Lock()