Merge pull request #1925 from molecula/rank-cache-bulk-invalidation

[FB-1206] Periodically invalidate rank cache during bulk add

cache.go

@@ -194,6 +194,14 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	}
 
 	c.entries[id] = n
+
+	// FB-1206: Periodically invalidate the cache when we are bulk loading
+	// as this can take up an upbounded amount of memory. This is especially
+	// true when restoring shards as all rows will be added.
+	if len(c.entries) > int(2*c.maxEntries) {
+		c.stats.Count(MetricRecalculateCache, 1, 1.0)
+		c.recalculate()
+	}
 }
 
 // Get returns a count for a given id.

cache_test.go

@@ -70,3 +70,15 @@ func TestCache_Rank_Dirty(t *testing.T) {
 		t.Fatalf("wrote %v but got %v", expect, got)
 	}
 }
+
+func TestCache_Rank_BulkAdd(t *testing.T) {
+	const cacheSize = 10
+	cache := pilosa.NewRankCache(uint32(cacheSize))
+
+	for i := uint64(0); i < 1000; i++ {
+		cache.BulkAdd(i, i)
+		if n := cache.Len(); n > cacheSize*2 {
+			t.Fatalf("entry count exceed 2x cache size: %d", n)
+		}
+	}
+}