clean up fragments more cleanly (in tests and benchmarks)

This commit is contained in:
Matt Jaffee 2018-12-11 15:24:49 -06:00
parent 4f808c2028
commit f2578c401f
No known key found for this signature in database
GPG key ID: 08A3DFFF987B11BF

View file

@ -49,7 +49,7 @@ func init() {
// Ensure a fragment can set a bit and retrieve it.
func TestFragment_SetBit(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set bits on the fragment.
if _, err := f.setBit(120, 1); err != nil {
@ -80,7 +80,7 @@ func TestFragment_SetBit(t *testing.T) {
// Ensure a fragment can clear a set bit.
func TestFragment_ClearBit(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set and then clear bits on the fragment.
if _, err := f.setBit(1000, 1); err != nil {
@ -107,7 +107,7 @@ func TestFragment_ClearBit(t *testing.T) {
// Ensure a fragment can clear a row.
func TestFragment_ClearRow(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set and then clear bits on the fragment.
if _, err := f.setBit(1000, 1); err != nil {
@ -134,7 +134,7 @@ func TestFragment_ClearRow(t *testing.T) {
// Ensure a fragment can set a row.
func TestFragment_SetRow(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 7, "")
defer f.Clean()
defer f.Clean(t)
rowID := uint64(1000)
@ -183,7 +183,7 @@ func TestFragment_SetRow(t *testing.T) {
func TestFragment_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set value.
if changed, err := f.setValue(100, 16, 3829); err != nil {
@ -211,7 +211,7 @@ func TestFragment_SetValue(t *testing.T) {
t.Run("Overwrite", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set value.
if changed, err := f.setValue(100, 16, 3829); err != nil {
@ -239,7 +239,7 @@ func TestFragment_SetValue(t *testing.T) {
t.Run("Clear", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set value.
if changed, err := f.setValue(100, 16, 3829); err != nil {
@ -267,7 +267,7 @@ func TestFragment_SetValue(t *testing.T) {
t.Run("NotExists", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set value.
if changed, err := f.setValue(100, 10, 20); err != nil {
@ -297,7 +297,7 @@ func TestFragment_SetValue(t *testing.T) {
}
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set values.
m := make(map[uint64]int64)
@ -335,7 +335,7 @@ func TestFragment_Sum(t *testing.T) {
const bitDepth = 16
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -374,7 +374,7 @@ func TestFragment_MinMax(t *testing.T) {
const bitDepth = 16
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -448,7 +448,7 @@ func TestFragment_Range(t *testing.T) {
t.Run("EQ", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -471,7 +471,7 @@ func TestFragment_Range(t *testing.T) {
t.Run("NEQ", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -494,7 +494,7 @@ func TestFragment_Range(t *testing.T) {
t.Run("LT", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -542,7 +542,7 @@ func TestFragment_Range(t *testing.T) {
t.Run("GT", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -590,7 +590,7 @@ func TestFragment_Range(t *testing.T) {
t.Run("BETWEEN", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -640,7 +640,7 @@ func TestFragment_Range(t *testing.T) {
// Ensure a fragment can snapshot correctly.
func TestFragment_Snapshot(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set and then clear bits on the fragment.
if _, err := f.setBit(1000, 1); err != nil {
@ -669,7 +669,7 @@ func TestFragment_Snapshot(t *testing.T) {
// Ensure a fragment can iterate over all bits in order.
func TestFragment_ForEachBit(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set bits on the fragment.
if _, err := f.setBit(100, 20); err != nil {
@ -698,7 +698,7 @@ func TestFragment_ForEachBit(t *testing.T) {
// Ensure a fragment can return the top n results.
func TestFragment_Top(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
// Set bits on the rows 100, 101, & 102.
f.mustSetBits(100, 1, 3, 200)
f.mustSetBits(101, 1)
@ -720,7 +720,7 @@ func TestFragment_Top(t *testing.T) {
// Ensure a fragment can filter rows when retrieving the top n rows.
func TestFragment_Top_Filter(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
// Set bits on the rows 100, 101, & 102.
f.mustSetBits(100, 1, 3, 200)
@ -750,7 +750,7 @@ func TestFragment_Top_Filter(t *testing.T) {
// Ensure a fragment can return top rows that intersect with an input row.
func TestFragment_TopN_Intersect(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
// Create an intersecting input row.
src := NewRow(1, 2, 3)
@ -781,7 +781,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
}
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
// Create an intersecting input row.
src := NewRow(
@ -819,7 +819,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
// Ensure a fragment can return top rows when specified by ID.
func TestFragment_TopN_IDs(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
// Set bits on various rows.
f.mustSetBits(100, 1, 2, 3)
@ -840,7 +840,7 @@ func TestFragment_TopN_IDs(t *testing.T) {
// Ensure a fragment return none if CacheTypeNone is set
func TestFragment_TopN_NopCache(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeNone)
defer f.Clean()
defer f.Clean(t)
// Set bits on various rows.
f.mustSetBits(100, 1, 2, 3)
@ -888,7 +888,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
if err := f.Open(); err != nil {
panic(err)
}
defer f.Clean()
defer f.Clean(t)
// Set bits on various rows.
f.mustSetBits(100, 1, 2, 3)
@ -921,7 +921,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
// Ensure fragment can return a checksum for its blocks.
func TestFragment_Checksum(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Retrieve checksum and set bits.
orig := f.Checksum()
@ -940,7 +940,7 @@ func TestFragment_Checksum(t *testing.T) {
// Ensure fragment can return a checksum for a given block.
func TestFragment_Blocks(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Retrieve initial checksum.
var prev []FragmentBlock
@ -978,7 +978,7 @@ func TestFragment_Blocks(t *testing.T) {
// Ensure fragment returns an empty checksum if no data exists for a block.
func TestFragment_Blocks_Empty(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set bits on a different block.
if _, err := f.setBit(100, 1); err != nil {
@ -996,7 +996,7 @@ func TestFragment_Blocks_Empty(t *testing.T) {
// Ensure a fragment's cache can be persisted between restarts.
func TestFragment_LRUCache_Persistence(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeLRU)
defer f.Clean()
defer f.Clean(t)
// Set bits on the fragment.
for i := uint64(0); i < 1000; i++ {
@ -1081,7 +1081,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
// Ensure a fragment can be copied to another fragment.
func TestFragment_WriteTo_ReadFrom(t *testing.T) {
f0 := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f0.Clean()
defer f0.Clean(t)
// Set and then clear bits on the fragment.
if _, err := f0.setBit(1000, 1); err != nil {
@ -1142,7 +1142,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
if err := f.Open(); err != nil {
b.Fatal(err)
}
defer f.Clean()
defer f.Clean(b)
// Reset timer and execute benchmark.
b.ResetTimer()
@ -1155,7 +1155,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
func BenchmarkFragment_IntersectionCount(b *testing.B) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(b)
f.MaxOpN = math.MaxInt32
// Generate some intersecting data.
@ -1186,7 +1186,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
func TestFragment_Tanimoto(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
src := NewRow(1, 2, 3)
@ -1209,7 +1209,7 @@ func TestFragment_Tanimoto(t *testing.T) {
func TestFragment_Zero_Tanimoto(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
src := NewRow(1, 2, 3)
@ -1234,7 +1234,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
func TestFragment_Snapshot_Run(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set bits on the fragment.
for i := uint64(1); i < 3; i++ {
@ -1261,7 +1261,7 @@ func TestFragment_Snapshot_Run(t *testing.T) {
// Ensure a fragment can set mutually exclusive values.
func TestFragment_SetMutex(t *testing.T) {
f := mustOpenMutexFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
var cols []uint64
@ -1375,7 +1375,7 @@ func TestFragment_ImportSet(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set import.
err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{})
@ -1411,7 +1411,7 @@ func TestFragment_ImportSet(t *testing.T) {
func TestFragment_ConcurrentImport(t *testing.T) {
t.Run("bulkImportStandard", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
eg := errgroup.Group{}
eg.Go(func() error { return f.bulkImportStandard([]uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) })
@ -1508,7 +1508,7 @@ func TestFragment_ImportMutex(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) {
f := mustOpenMutexFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set import.
err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{})
@ -1627,7 +1627,7 @@ func TestFragment_ImportBool(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) {
f := mustOpenBoolFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
// Set import.
err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{})
@ -1671,7 +1671,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
if err := f.Open(); err != nil {
b.Fatal(err)
}
defer f.Clean()
defer f.Clean(b)
b.ResetTimer()
// Reset timer and execute benchmark.
@ -1687,7 +1687,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
func BenchmarkFragment_FullSnapshot(b *testing.B) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(b)
// Generate some intersecting data.
maxX := 1048576 / 2
sz := maxX
@ -1725,7 +1725,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
func BenchmarkFragment_Import(b *testing.B) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(b)
maxX := 1048576 * 5 * 2
sz := maxX
rows := make([]uint64, sz)
@ -1771,11 +1771,11 @@ func BenchmarkImportRoaring(b *testing.B) {
b.StartTimer()
err := f.importRoaring(data, false)
if err != nil {
f.Clean()
f.Clean(b)
b.Fatalf("import error: %v", err)
}
b.StopTimer()
f.Clean()
f.Clean(b)
}
})
}
@ -1808,7 +1808,7 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) {
}
b.StopTimer()
for j := 0; j < concurrency; j++ {
frags[j].Clean()
frags[j].Clean(b)
}
}
})
@ -1843,7 +1843,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
}
b.StopTimer()
for j := 0; j < concurrency; j++ {
frags[j].Clean()
frags[j].Clean(b)
}
}
})
@ -1866,7 +1866,7 @@ func BenchmarkImportStandard(b *testing.B) {
b.Errorf("import error: %v", err)
}
b.StopTimer()
f.Clean()
f.Clean(b)
}
})
}
@ -1894,13 +1894,13 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
b.StartTimer()
err = f.importRoaring(updata, false)
if err != nil {
f.Clean()
f.Clean(b)
b.Errorf("import error: %v", err)
}
b.StopTimer()
stat, _ := f.file.Stat()
fileSize[name] = stat.Size()
f.Clean()
f.Clean(b)
}
})
@ -2017,16 +2017,17 @@ func BenchmarkFileWrite(b *testing.B) {
/////////////////////////////////////////////////////////////////////
func (f *fragment) Clean() error {
func (f *fragment) Clean(t testing.TB) {
errc := f.Close()
errf := os.Remove(f.path)
errp := os.Remove(f.cachePath())
if errc != nil {
return errc
} else if errf != nil {
return errf
if errc != nil || errf != nil {
t.Fatal("cleaning up fragment: ", errc, errf, errp)
}
// not all fragments have cache files
if errp != nil && !os.IsNotExist(errp) {
t.Fatalf("cleaning up fragment cache: %v", errp)
}
return errp
}
// mustOpenFragment returns a new instance of Fragment with a temporary path.
@ -2092,7 +2093,7 @@ func (f *fragment) mustSetBits(rowID uint64, columnIDs ...uint64) {
func TestFragment_RowsIteration(t *testing.T) {
t.Run("firstContainer", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
expectedAll := make([]uint64, 0)
expectedOdd := make([]uint64, 0)
@ -2119,7 +2120,7 @@ func TestFragment_RowsIteration(t *testing.T) {
t.Run("secondRow", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
expected := []uint64{1, 2}
if _, err := f.setBit(1, 66000); err != nil {
@ -2143,7 +2144,7 @@ func TestFragment_RowsIteration(t *testing.T) {
t.Run("combinations", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
expectedRows := make([]uint64, 0)
for r := uint64(1); r < uint64(10000); r += 100 {
@ -2190,7 +2191,7 @@ func TestFragment_RoaringImport(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean()
defer f.Clean(t)
for num, input := range test {
buf := &bytes.Buffer{}
bm := roaring.NewBitmap(input...)
@ -2233,7 +2234,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
options := &ImportOptions{}
err := f.bulkImport(test.rowIDs, test.colIDs, options)
@ -2367,7 +2368,7 @@ func calcExpected(inputs ...[]uint64) [][]uint64 {
func TestFragmentRowIterator(t *testing.T) {
t.Run("basic", func(t *testing.T) {
f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
f.mustSetBits(0, 0)
f.mustSetBits(1, 0)
f.mustSetBits(2, 0)
@ -2400,7 +2401,7 @@ func TestFragmentRowIterator(t *testing.T) {
t.Run("skipped rows", func(t *testing.T) {
f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
f.mustSetBits(1, 0)
f.mustSetBits(3, 0)
f.mustSetBits(5, 0)
@ -2433,7 +2434,7 @@ func TestFragmentRowIterator(t *testing.T) {
t.Run("basic wrapped", func(t *testing.T) {
f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
f.mustSetBits(0, 0)
f.mustSetBits(1, 0)
f.mustSetBits(2, 0)
@ -2458,7 +2459,7 @@ func TestFragmentRowIterator(t *testing.T) {
t.Run("skipped rows wrapped", func(t *testing.T) {
f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
defer f.Clean()
defer f.Clean(t)
f.mustSetBits(1, 0)
f.mustSetBits(3, 0)
f.mustSetBits(5, 0)