scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 09:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

don't fill up empty space with non-functional ops logs

Browse source 
Two changes:

1. Don't write batch/roaring adds or removes when N is 0, because
a write of no bits is not a meaningful write.
2. When unmarshalling roaring things, if a roaring bitmap didn't
change many bits, treat it as having changed at least 1 bit per 8 bytes,
so an 8KB hunk of roaring data counts as 1K changes, which will
nudge us towards snapshotting. This should keep us from having
Large Files show up so much.

This was particularly noticeable on the existence field, which
tends to a steady state of "completely full" very quickly in a lot
of cases.
This commit is contained in:
Seebs 2020-03-04 09:55:01 -06:00
parent 53c486fce0
commit eb263b7666
1 changed files with 8 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
roaring/roaring.go
View file

@ -297,7 +297,7 @@ func (b *Bitmap) AddN(a ...uint64) (changed int, err error) {
changed = b.DirectAddN(a...) // modifies a in-place
if b.OpWriter != nil {
if b.OpWriter != nil && changed > 0 {
op := &op{
typ: opTypeAddBatch,
values: a[:changed],
@ -404,7 +404,7 @@ func (b *Bitmap) RemoveN(a ...uint64) (changed int, err error) {
changed = b.DirectRemoveN(a...) // modifies a in-place
if b.OpWriter != nil {
if b.OpWriter != nil && changed > 0 {
op := &op{
typ: opTypeRemoveBatch,
values: a[:changed],
@ -1727,7 +1727,7 @@ func (b *Bitmap) ImportRoaringBits(data []byte, clear bool, log bool, rowSize ui
return changed, rowSet, itrErr
}
err = nil
if log {
if log && changed > 0 {
op := op{opN: changed, roaring: data}
if clear {
op.typ = opTypeRemoveRoaring
@ -4820,6 +4820,11 @@ func (op *op) UnmarshalBinary(data []byte) error {
return fmt.Errorf("op data truncated - expected %d, got %d", 13+op.value, len(data))
}
op.opN = int(binary.LittleEndian.Uint32(data[13:17]))
// gratuitous hack: treat any roaring write as having at least 1/8 of
// its length in bits, even if it didn't actually change things.
if op.opN < int(op.value/8) {
op.opN = int(op.value / 8)
}
op.roaring = data[17 : 17+op.value]
_, _ = h.Write(data[13 : 17+op.value])
// op.value = 0