scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 17:15:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

always treat BSI fields as having at least their depth

Browse source 
If you imported only small values, BSI fields could end up
not bothering to clear higher bits in existing values, which
produced strange behaviors.

We also move the computation of requiredDepth, and the change
to the field, down, combining it with the other checks of the
values for min/max being in range.

Without this, a data set with a ludicrously large value in it
could break a BSI field's depth even though the import would then
reject it.
This commit is contained in:
Seebs 2019-11-11 12:40:14 -06:00
parent 1a44f02e3c
commit a804a0dfb1
1 changed files with 31 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
field.go
View file

@ -1409,34 +1409,12 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
return errors.Wrap(ErrBSIGroupNotFound, f.name)
}
// Find the lowest/highest values.
// We want to determine the required bit depth, in case the field doesn't
// have as many bits currently as would be needed to represent these values,
// but only if the values are in-range for the field.
var min, max int64
for i, value := range values {
if i == 0 || value < min {
min = value
}
if i == 0 || value > max {
max = value
}
}
// Determine the highest bit depth required by the min & max.
requiredDepth := bitDepthInt64(min - bsig.Base)
if v := bitDepthInt64(max - bsig.Base); v > requiredDepth {
requiredDepth = v
}
// Increase bit depth if required.
if requiredDepth > bsig.BitDepth {
if err := func() error {
f.mu.Lock()
defer f.mu.Unlock()
bsig.BitDepth = requiredDepth
f.options.BitDepth = requiredDepth
return f.saveMeta()
}(); err != nil {
return errors.Wrap(err, "increasing bsi bit depth")
}
if len(values) > 0 {
min, max = values[0], values[0]
}
// Split import data by fragment.
@ -1448,6 +1426,12 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
} else if value < bsig.Min {
return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooLow, columnID, value)
}
if value > max {
max = value
}
if value < min {
min = value
}
// Attach value to each bsiGroup view.
for _, name := range []string{viewName} {
@ -1459,6 +1443,26 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
}
}
// Determine the highest bit depth required by the min & max.
requiredDepth := bitDepthInt64(min - bsig.Base)
if v := bitDepthInt64(max - bsig.Base); v > requiredDepth {
requiredDepth = v
}
// Increase bit depth if required.
if requiredDepth > bsig.BitDepth {
if err := func() error {
f.mu.Lock()
defer f.mu.Unlock()
bsig.BitDepth = requiredDepth
f.options.BitDepth = requiredDepth
return f.saveMeta()
}(); err != nil {
return errors.Wrap(err, "increasing bsi bit depth")
}
} else {
requiredDepth = bsig.BitDepth
}
// Import into each fragment.
for key, data := range dataByFragment {
// The view must already exist (i.e. we can't create it)