scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Import optimization

Browse source
This commit is contained in:
Todd Gruben 2016-06-01 14:10:34 -05:00
parent 9c5b4829e5
commit 3d4eaecac0
1 changed files with 18 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
fragment.go
View file

@ -821,9 +821,11 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error {
// Process every bit.
// If an error occurs then reopen the storage.
if err := func() error {
last_id := uint64(0)
bm_counter := uint64(0)
var bitmap *Bitmap
for i := range bitmapIDs {
bitmapID, profileID := bitmapIDs[i], profileIDs[i]
// Determine the position of the bit in the storage.
pos, err := f.pos(bitmapID, profileID)
if err != nil {
@ -835,11 +837,24 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error {
return err
}
// import optimization to avoid linear foreach calls
// slight risk of concurrent cache counter being off but
// no real danger
if bitmapID != last_id {
bitmap = f.bitmap(bitmapID)
if last_id != 0 {
f.cache.Add(last_id, bm_counter)
}
bm_counter = bitmap.Count()
last_id = bitmapID
}
// Invalidate block checksum.
delete(f.checksums, int(bitmapID/HashBlockSize))
if bitmap.SetBit(profileID) {
bm_counter += 1
}
// Update the cache.
f.bitmap(bitmapID).SetBit(profileID)
}
return nil
}(); err != nil {