diff --git a/fragment_internal_test.go b/fragment_internal_test.go index fd6da6405..b0864ae4a 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -49,7 +49,7 @@ func init() { // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set bits on the fragment. if _, err := f.setBit(120, 1); err != nil { @@ -80,7 +80,7 @@ func TestFragment_SetBit(t *testing.T) { // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -107,7 +107,7 @@ func TestFragment_ClearBit(t *testing.T) { // Ensure a fragment can clear a row. func TestFragment_ClearRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -134,7 +134,7 @@ func TestFragment_ClearRow(t *testing.T) { // Ensure a fragment can set a row. func TestFragment_SetRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 7, "") - defer f.Clean() + defer f.Clean(t) rowID := uint64(1000) @@ -183,7 +183,7 @@ func TestFragment_SetRow(t *testing.T) { func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -211,7 +211,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("Overwrite", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -239,7 +239,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("Clear", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -267,7 +267,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("NotExists", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 10, 20); err != nil { @@ -297,7 +297,7 @@ func TestFragment_SetValue(t *testing.T) { } f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. m := make(map[uint64]int64) @@ -335,7 +335,7 @@ func TestFragment_Sum(t *testing.T) { const bitDepth = 16 f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -374,7 +374,7 @@ func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -448,7 +448,7 @@ func TestFragment_Range(t *testing.T) { t.Run("EQ", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -471,7 +471,7 @@ func TestFragment_Range(t *testing.T) { t.Run("NEQ", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -494,7 +494,7 @@ func TestFragment_Range(t *testing.T) { t.Run("LT", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -542,7 +542,7 @@ func TestFragment_Range(t *testing.T) { t.Run("GT", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -590,7 +590,7 @@ func TestFragment_Range(t *testing.T) { t.Run("BETWEEN", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -640,7 +640,7 @@ func TestFragment_Range(t *testing.T) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -669,7 +669,7 @@ func TestFragment_Snapshot(t *testing.T) { // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set bits on the fragment. if _, err := f.setBit(100, 20); err != nil { @@ -698,7 +698,7 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) // Set bits on the rows 100, 101, & 102. f.mustSetBits(100, 1, 3, 200) f.mustSetBits(101, 1) @@ -720,7 +720,7 @@ func TestFragment_Top(t *testing.T) { // Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) // Set bits on the rows 100, 101, & 102. f.mustSetBits(100, 1, 3, 200) @@ -750,7 +750,7 @@ func TestFragment_Top_Filter(t *testing.T) { // Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) // Create an intersecting input row. src := NewRow(1, 2, 3) @@ -781,7 +781,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { } f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) // Create an intersecting input row. src := NewRow( @@ -819,7 +819,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -840,7 +840,7 @@ func TestFragment_TopN_IDs(t *testing.T) { // Ensure a fragment return none if CacheTypeNone is set func TestFragment_TopN_NopCache(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeNone) - defer f.Clean() + defer f.Clean(t) // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -888,7 +888,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { if err := f.Open(); err != nil { panic(err) } - defer f.Clean() + defer f.Clean(t) // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -921,7 +921,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Retrieve checksum and set bits. orig := f.Checksum() @@ -940,7 +940,7 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Retrieve initial checksum. var prev []FragmentBlock @@ -978,7 +978,7 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set bits on a different block. if _, err := f.setBit(100, 1); err != nil { @@ -996,7 +996,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeLRU) - defer f.Clean() + defer f.Clean(t) // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { @@ -1081,7 +1081,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { f0 := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f0.Clean() + defer f0.Clean(t) // Set and then clear bits on the fragment. if _, err := f0.setBit(1000, 1); err != nil { @@ -1142,7 +1142,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Clean() + defer f.Clean(b) // Reset timer and execute benchmark. b.ResetTimer() @@ -1155,7 +1155,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { func BenchmarkFragment_IntersectionCount(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(b) f.MaxOpN = math.MaxInt32 // Generate some intersecting data. @@ -1186,7 +1186,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { func TestFragment_Tanimoto(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) src := NewRow(1, 2, 3) @@ -1209,7 +1209,7 @@ func TestFragment_Tanimoto(t *testing.T) { func TestFragment_Zero_Tanimoto(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) src := NewRow(1, 2, 3) @@ -1234,7 +1234,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { func TestFragment_Snapshot_Run(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set bits on the fragment. for i := uint64(1); i < 3; i++ { @@ -1261,7 +1261,7 @@ func TestFragment_Snapshot_Run(t *testing.T) { // Ensure a fragment can set mutually exclusive values. func TestFragment_SetMutex(t *testing.T) { f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) var cols []uint64 @@ -1375,7 +1375,7 @@ func TestFragment_ImportSet(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1411,7 +1411,7 @@ func TestFragment_ImportSet(t *testing.T) { func TestFragment_ConcurrentImport(t *testing.T) { t.Run("bulkImportStandard", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) eg := errgroup.Group{} eg.Go(func() error { return f.bulkImportStandard([]uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) }) @@ -1508,7 +1508,7 @@ func TestFragment_ImportMutex(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1627,7 +1627,7 @@ func TestFragment_ImportBool(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { f := mustOpenBoolFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1671,7 +1671,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Clean() + defer f.Clean(b) b.ResetTimer() // Reset timer and execute benchmark. @@ -1687,7 +1687,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { func BenchmarkFragment_FullSnapshot(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(b) // Generate some intersecting data. maxX := 1048576 / 2 sz := maxX @@ -1725,7 +1725,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { func BenchmarkFragment_Import(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(b) maxX := 1048576 * 5 * 2 sz := maxX rows := make([]uint64, sz) @@ -1771,11 +1771,11 @@ func BenchmarkImportRoaring(b *testing.B) { b.StartTimer() err := f.importRoaring(data, false) if err != nil { - f.Clean() + f.Clean(b) b.Fatalf("import error: %v", err) } b.StopTimer() - f.Clean() + f.Clean(b) } }) } @@ -1808,7 +1808,7 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) { } b.StopTimer() for j := 0; j < concurrency; j++ { - frags[j].Clean() + frags[j].Clean(b) } } }) @@ -1843,7 +1843,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { } b.StopTimer() for j := 0; j < concurrency; j++ { - frags[j].Clean() + frags[j].Clean(b) } } }) @@ -1866,7 +1866,7 @@ func BenchmarkImportStandard(b *testing.B) { b.Errorf("import error: %v", err) } b.StopTimer() - f.Clean() + f.Clean(b) } }) } @@ -1894,13 +1894,13 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { b.StartTimer() err = f.importRoaring(updata, false) if err != nil { - f.Clean() + f.Clean(b) b.Errorf("import error: %v", err) } b.StopTimer() stat, _ := f.file.Stat() fileSize[name] = stat.Size() - f.Clean() + f.Clean(b) } }) @@ -2017,16 +2017,17 @@ func BenchmarkFileWrite(b *testing.B) { ///////////////////////////////////////////////////////////////////// -func (f *fragment) Clean() error { +func (f *fragment) Clean(t testing.TB) { errc := f.Close() errf := os.Remove(f.path) errp := os.Remove(f.cachePath()) - if errc != nil { - return errc - } else if errf != nil { - return errf + if errc != nil || errf != nil { + t.Fatal("cleaning up fragment: ", errc, errf, errp) + } + // not all fragments have cache files + if errp != nil && !os.IsNotExist(errp) { + t.Fatalf("cleaning up fragment cache: %v", errp) } - return errp } // mustOpenFragment returns a new instance of Fragment with a temporary path. @@ -2092,7 +2093,7 @@ func (f *fragment) mustSetBits(rowID uint64, columnIDs ...uint64) { func TestFragment_RowsIteration(t *testing.T) { t.Run("firstContainer", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) expectedAll := make([]uint64, 0) expectedOdd := make([]uint64, 0) @@ -2119,7 +2120,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("secondRow", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) expected := []uint64{1, 2} if _, err := f.setBit(1, 66000); err != nil { @@ -2143,7 +2144,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("combinations", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) expectedRows := make([]uint64, 0) for r := uint64(1); r < uint64(10000); r += 100 { @@ -2190,7 +2191,7 @@ func TestFragment_RoaringImport(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) for num, input := range test { buf := &bytes.Buffer{} bm := roaring.NewBitmap(input...) @@ -2233,7 +2234,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) options := &ImportOptions{} err := f.bulkImport(test.rowIDs, test.colIDs, options) @@ -2367,7 +2368,7 @@ func calcExpected(inputs ...[]uint64) [][]uint64 { func TestFragmentRowIterator(t *testing.T) { t.Run("basic", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) f.mustSetBits(0, 0) f.mustSetBits(1, 0) f.mustSetBits(2, 0) @@ -2400,7 +2401,7 @@ func TestFragmentRowIterator(t *testing.T) { t.Run("skipped rows", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) f.mustSetBits(1, 0) f.mustSetBits(3, 0) f.mustSetBits(5, 0) @@ -2433,7 +2434,7 @@ func TestFragmentRowIterator(t *testing.T) { t.Run("basic wrapped", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) f.mustSetBits(0, 0) f.mustSetBits(1, 0) f.mustSetBits(2, 0) @@ -2458,7 +2459,7 @@ func TestFragmentRowIterator(t *testing.T) { t.Run("skipped rows wrapped", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) f.mustSetBits(1, 0) f.mustSetBits(3, 0) f.mustSetBits(5, 0)