scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 17:15:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

move requiredDepth calculation after min/max ranges are checked

Browse source 
Without this, a data set with a ludicrously large value in it
could break a BSI field's depth even though the import would then
reject it.

always treat BSI fields as having at least their depth:

If you imported only small values, BSI fields could end up
not bothering to clear higher bits in existing values, which
produced strange behaviors.
This commit is contained in:
Travis Turner 2019-11-11 16:48:14 -06:00
parent 5d71f04e54
commit 5ebd7e8da8
No known key found for this signature in database
GPG key ID: 37080CC2042BA34E
1 changed files with 31 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
field.go
View file

@ -1269,34 +1269,12 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
return errors.Wrap(ErrBSIGroupNotFound, f.name)
}
// Find the lowest/highest values.
// We want to determine the required bit depth, in case the field doesn't
// have as many bits currently as would be needed to represent these values,
// but only if the values are in-range for the field.
var min, max int64
for i, value := range values {
if i == 0 || value < min {
min = value
}
if i == 0 || value > max {
max = value
}
}
// Determine the highest bit depth required by the min & max.
requiredDepth := bitDepthInt64(min - bsig.Base)
if v := bitDepthInt64(max - bsig.Base); v > requiredDepth {
requiredDepth = v
}
// Increase bit depth if required.
if requiredDepth > bsig.BitDepth {
if err := func() error {
f.mu.Lock()
defer f.mu.Unlock()
bsig.BitDepth = requiredDepth
f.options.BitDepth = requiredDepth
return f.saveMeta()
}(); err != nil {
return errors.Wrap(err, "increasing bsi bit depth")
}
if len(values) > 0 {
min, max = values[0], values[0]
}
// Split import data by fragment.
@ -1308,6 +1286,12 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
} else if value < bsig.Min {
return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooLow, columnID, value)
}
if value > max {
max = value
}
if value < min {
min = value
}
// Attach value to each bsiGroup view.
for _, name := range []string{viewName} {
@ -1319,6 +1303,26 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
}
}
// Determine the highest bit depth required by the min & max.
requiredDepth := bitDepthInt64(min - bsig.Base)
if v := bitDepthInt64(max - bsig.Base); v > requiredDepth {
requiredDepth = v
}
// Increase bit depth if required.
if requiredDepth > bsig.BitDepth {
if err := func() error {
f.mu.Lock()
defer f.mu.Unlock()
bsig.BitDepth = requiredDepth
f.options.BitDepth = requiredDepth
return f.saveMeta()
}(); err != nil {
return errors.Wrap(err, "increasing bsi bit depth")
}
} else {
requiredDepth = bsig.BitDepth
}
// Import into each fragment.
for key, data := range dataByFragment {
// The view must already exist (i.e. we can't create it)