From 76e747055945e39fc6f151fbf3099ffaf19dfdc0 Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 31 Mar 2020 12:02:45 -0500 Subject: [PATCH] make mergeBlock use transactions mergeBlock was bypassing the transaction setup stuff, which means that if we ran out of open files, mergeBlock wouldn't generate ops log entries (!), also it didn't update the cache (!). This came up because it also didn't enjoy the "catch your segfaults and issue a diagnostic" behavior offered by the generation code. Switch to computing positions directly and calling importPositions, which does a transaction. --- fragment.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/fragment.go b/fragment.go index f3c4d503b..28c524025 100644 --- a/fragment.go +++ b/fragment.go @@ -1862,21 +1862,20 @@ func (f *fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e } } - // Set local bits. + rowSet := make(map[uint64]struct{}, len(sets[0].columnIDs)) + // compute positions directly, replacing columnIDs with the computed + // positions for i := range sets[0].columnIDs { - if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.shard*ShardWidth)+sets[0].columnIDs[i]); err != nil { - return nil, nil, errors.Wrap(err, "setting") - } + rowSet[sets[0].rowIDs[i]] = struct{}{} + sets[0].columnIDs[i] += sets[0].rowIDs[i] * ShardWidth } - - // Clear local bits. for i := range clears[0].columnIDs { - if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.shard*ShardWidth)+clears[0].columnIDs[i]); err != nil { - return nil, nil, errors.Wrap(err, "clearing") - } + rowSet[clears[0].rowIDs[i]] = struct{}{} + clears[0].columnIDs[i] += clears[0].rowIDs[i] * ShardWidth } + err = f.importPositions(sets[0].columnIDs, clears[0].columnIDs, rowSet) - return sets[1:], clears[1:], nil + return sets[1:], clears[1:], err } // bulkImport bulk imports a set of bits and then snapshots the storage.