diff --git a/fragment.go b/fragment.go
index 0801c8001..32b870fb9 100644
--- a/fragment.go
+++ b/fragment.go
@@ -1763,11 +1763,8 @@ func (f *fragment) importRoaring(data []byte, clear bool) error {
 
 	if clear {
 		bm = f.storage.Difference(bm)
-	} else if f.storage.Containers.Size() >= bm.Containers.Size() {
-		f.storage.UnionInPlace(bm)
-		bm = f.storage
-	} else {
-		bm.UnionInPlace(f.storage)
+	} else if f.storage.Any() {
+		bm = f.storage.Union(bm)
 	}
 
 	for rowID := range rowSet {