scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix endpoint tests + change BitDepth type to uint64

Browse source
This commit is contained in:
Kuba Podgórski 2021-02-11 18:05:18 +01:00 • committed by Travis
parent a6297dc48e
commit 2f35b51db8
No known key found for this signature in database
GPG key ID: 37080CC2042BA34E
12 changed files with 96 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
encoding/proto/proto.go
View file

@ -1046,7 +1046,7 @@ func (s Serializer) decodeFieldOptions(options *internal.FieldOptions, m *pilosa
s.decodeDecimal(options.Max, &m.Max)
m.Base = options.Base
m.Scale = options.Scale
m.BitDepth = uint(options.BitDepth)
m.BitDepth = uint64(options.BitDepth)
m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum)
m.Keys = options.Keys
m.ForeignIndex = options.ForeignIndex

2
executor.go
View file

@ -4086,7 +4086,7 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri
mergeBits(sign, 1<<63, data)
// Copy in the significand.
for i := uint(0); i < bsig.BitDepth; i++ {
for i := uint64(0); i < bsig.BitDepth; i++ {
bits, err := fragment.row(tx, bsiOffsetBit+uint64(i))
if err != nil {
return ExtractedIDMatrix{}, errors.Wrap(err, "loading BSI significand bit from fragment")

20
field.go
View file

@ -844,7 +844,7 @@ func (f *Field) loadMeta() error {
f.options.Max = max
f.options.Base = pb.Base
f.options.Scale = pb.Scale
f.options.BitDepth = uint(pb.BitDepth)
f.options.BitDepth = pb.BitDepth
f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum)
f.options.Keys = pb.Keys
f.options.NoStandardView = pb.NoStandardView
@ -1871,9 +1871,9 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte,
return err
}
var bitDepth uint
var bitDepth uint64
if maxRowID+1 > bsiOffsetBit {
bitDepth = uint(maxRowID + 1 - bsiOffsetBit)
bitDepth = uint64(maxRowID + 1 - bsiOffsetBit)
}
bsig := f.bsiGroup(f.name)
@ -1915,7 +1915,7 @@ func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FieldOptions represents options to set when initializing a field.
type FieldOptions struct {
Base int64 `json:"base,omitempty"`
BitDepth uint `json:"bitDepth,omitempty"`
BitDepth uint64 `json:"bitDepth,omitempty"`
Min pql.Decimal `json:"min,omitempty"`
Max pql.Decimal `json:"max,omitempty"`
Scale int64 `json:"scale,omitempty"`
@ -2009,7 +2009,7 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
Type string `json:"type"`
Base int64 `json:"base"`
BitDepth uint `json:"bitDepth"`
BitDepth uint64 `json:"bitDepth"`
Min pql.Decimal `json:"min"`
Max pql.Decimal `json:"max"`
Keys bool `json:"keys"`
@ -2028,7 +2028,7 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
Type string `json:"type"`
Base int64 `json:"base"`
Scale int64 `json:"scale"`
BitDepth uint `json:"bitDepth"`
BitDepth uint64 `json:"bitDepth"`
Min pql.Decimal `json:"min"`
Max pql.Decimal `json:"max"`
Keys bool `json:"keys"`
@ -2109,7 +2109,7 @@ type bsiGroup struct {
Max int64 `json:"max,omitempty"`
Base int64 `json:"base,omitempty"`
Scale int64 `json:"scale,omitempty"`
BitDepth uint `json:"bitDepth,omitempty"`
BitDepth uint64 `json:"bitDepth,omitempty"`
}
// baseValue adjusts the value to align with the range for Field for a certain
@ -2209,12 +2209,12 @@ func isValidCacheType(v string) bool {
}
// bitDepth returns the number of bits required to store a value.
func bitDepth(v uint64) uint {
return uint(bits.Len64(v))
func bitDepth(v uint64) uint64 {
return uint64(bits.Len64(v))
}
// bitDepthInt64 returns the required bit depth for abs(v).
func bitDepthInt64(v int64) uint {
func bitDepthInt64(v int64) uint64 {
if v < 0 {
return bitDepth(uint64(-v))
}

72
fragment.go
View file

@ -965,7 +965,7 @@ func (f *fragment) bit(tx Tx, rowID, columnID uint64) (bool, error) {
}
// value uses a column of bits to read a multi-bit value.
func (f *fragment) value(tx Tx, columnID uint64, bitDepth uint) (value int64, exists bool, err error) {
func (f *fragment) value(tx Tx, columnID uint64, bitDepth uint64) (value int64, exists bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
@ -977,7 +977,7 @@ func (f *fragment) value(tx Tx, columnID uint64, bitDepth uint) (value int64, ex
}
// Compute other bits into a value.
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
if v, err := f.bit(tx, uint64(bsiOffsetBit+i), columnID); err != nil {
return 0, false, errors.Wrapf(err, "getting value bit %d", i)
} else if v {
@ -996,16 +996,16 @@ func (f *fragment) value(tx Tx, columnID uint64, bitDepth uint) (value int64, ex
}
// clearValue uses a column of bits to clear a multi-bit value.
func (f *fragment) clearValue(tx Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) {
func (f *fragment) clearValue(tx Tx, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) {
return f.setValueBase(tx, columnID, bitDepth, value, true)
}
// setValue uses a column of bits to set a multi-bit value.
func (f *fragment) setValue(tx Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) {
func (f *fragment) setValue(tx Tx, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) {
return f.setValueBase(tx, columnID, bitDepth, value, false)
}
func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64, clear bool, toSet, toClear []uint64) ([]uint64, []uint64, error) {
func (f *fragment) positionsForValue(columnID uint64, bitDepth uint64, value int64, clear bool, toSet, toClear []uint64) ([]uint64, []uint64, error) {
// Convert value to an unsigned representation.
uvalue := uint64(value)
if value < 0 {
@ -1030,7 +1030,7 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64
toSet = append(toSet, bit)
}
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
bit, err := f.pos(uint64(bsiOffsetBit+i), columnID)
if err != nil {
return toSet, toClear, errors.Wrap(err, "getting pos")
@ -1046,7 +1046,7 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64
}
// TODO get rid of this and use positionsForValue to generate a single write op, and set that with importPositions.
func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) {
func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint64, value int64, clear bool) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
@ -1073,7 +1073,7 @@ func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint, value
uvalue = uint64(-value)
}
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
if uvalue&(1<<i) != 0 {
if c, err := f.unprotectedSetBit(tx, uint64(bsiOffsetBit+i), columnID); err != nil {
return err
@ -1125,14 +1125,14 @@ func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint, value
}
// importSetValue is a more efficient SetValue just for imports.
func (f *fragment) importSetValue(txb *TxBitmap, columnID uint64, bitDepth uint, value int64, clear bool) (changed int, err error) { // nolint: unparam
func (f *fragment) importSetValue(txb *TxBitmap, columnID uint64, bitDepth uint64, value int64, clear bool) (changed int, err error) { // nolint: unparam
// Convert value to an unsigned representation.
uvalue := uint64(value)
if value < 0 {
uvalue = uint64(-value)
}
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
bit, err := f.pos(uint64(bsiOffsetBit+i), columnID)
if err != nil {
return changed, errors.Wrap(err, "getting pos")
@ -1194,7 +1194,7 @@ func (f *fragment) importSetValue(txb *TxBitmap, columnID uint64, bitDepth uint,
// sum returns the sum of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint) (sum int64, count uint64, err error) {
func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint64) (sum int64, count uint64, err error) {
// Compute count based on the existence row.
consider, err := f.row(tx, bsiExistsBit)
if err != nil {
@ -1225,7 +1225,7 @@ func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint) (sum int64, count uint
//
// Execute once for positive numbers and once for negative. Subtract the
// negative sum from the positive sum.
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
row, err := f.row(tx, uint64(bsiOffsetBit+i))
if err != nil {
return sum, count, err
@ -1243,7 +1243,7 @@ func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint) (sum int64, count uint
// min returns the min of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *fragment) min(tx Tx, filter *Row, bitDepth uint) (min int64, count uint64, err error) {
func (f *fragment) min(tx Tx, filter *Row, bitDepth uint64) (min int64, count uint64, err error) {
consider, err := f.row(tx, bsiExistsBit)
if err != nil {
return min, count, err
@ -1272,7 +1272,7 @@ func (f *fragment) min(tx Tx, filter *Row, bitDepth uint) (min int64, count uint
}
// minUnsigned the lowest value without considering the sign bit. Filter is required.
func (f *fragment) minUnsigned(tx Tx, filter *Row, bitDepth uint) (min int64, count uint64, err error) {
func (f *fragment) minUnsigned(tx Tx, filter *Row, bitDepth uint64) (min int64, count uint64, err error) {
for i := int(bitDepth - 1); i >= 0; i-- {
row, err := f.row(tx, uint64(bsiOffsetBit+i))
if err != nil {
@ -1294,7 +1294,7 @@ func (f *fragment) minUnsigned(tx Tx, filter *Row, bitDepth uint) (min int64, co
// max returns the max of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *fragment) max(tx Tx, filter *Row, bitDepth uint) (max int64, count uint64, err error) {
func (f *fragment) max(tx Tx, filter *Row, bitDepth uint64) (max int64, count uint64, err error) {
consider, err := f.row(tx, bsiExistsBit)
if err != nil {
return max, count, err
@ -1323,7 +1323,7 @@ func (f *fragment) max(tx Tx, filter *Row, bitDepth uint) (max int64, count uint
}
// maxUnsigned the highest value without considering the sign bit. Filter is required.
func (f *fragment) maxUnsigned(tx Tx, filter *Row, bitDepth uint) (max int64, count uint64, err error) {
func (f *fragment) maxUnsigned(tx Tx, filter *Row, bitDepth uint64) (max int64, count uint64, err error) {
for i := int(bitDepth - 1); i >= 0; i-- {
row, err := f.row(tx, uint64(bsiOffsetBit+i))
if err != nil {
@ -1424,7 +1424,7 @@ func (f *fragment) maxRowID(tx Tx) (_ uint64, err error) {
}
// rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate.
func (f *fragment) rangeOp(tx Tx, op pql.Token, bitDepth uint, predicate int64) (*Row, error) {
func (f *fragment) rangeOp(tx Tx, op pql.Token, bitDepth uint64, predicate int64) (*Row, error) {
switch op {
case pql.EQ:
return f.rangeEQ(tx, bitDepth, predicate)
@ -1450,7 +1450,7 @@ func absInt64(v int64) uint64 {
}
}
func (f *fragment) rangeEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error) {
func (f *fragment) rangeEQ(tx Tx, bitDepth uint64, predicate int64) (*Row, error) {
// Start with set of columns with values set.
b, err := f.row(tx, bsiExistsBit)
if err != nil {
@ -1458,7 +1458,7 @@ func (f *fragment) rangeEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error)
}
upredicate := absInt64(predicate)
if uint(bits.Len64(upredicate)) > bitDepth {
if uint64(bits.Len64(upredicate)) > bitDepth {
// Predicate is out of range.
return NewRow(), nil
}
@ -1492,7 +1492,7 @@ func (f *fragment) rangeEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error)
return b, nil
}
func (f *fragment) rangeNEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error) {
func (f *fragment) rangeNEQ(tx Tx, bitDepth uint64, predicate int64) (*Row, error) {
// Start with set of columns with values set.
b, err := f.row(tx, bsiExistsBit)
if err != nil {
@ -1511,7 +1511,7 @@ func (f *fragment) rangeNEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error)
return b, nil
}
func (f *fragment) rangeLT(tx Tx, bitDepth uint, predicate int64, allowEquality bool) (*Row, error) {
func (f *fragment) rangeLT(tx Tx, bitDepth uint64, predicate int64, allowEquality bool) (*Row, error) {
if predicate == 1 && !allowEquality {
predicate, allowEquality = 0, true
}
@ -1557,9 +1557,9 @@ func (f *fragment) rangeLT(tx Tx, bitDepth uint, predicate int64, allowEquality
}
// rangeLTUnsigned returns all bits LT/LTE the predicate without considering the sign bit.
func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint64, predicate uint64, allowEquality bool) (*Row, error) {
switch {
case uint(bits.Len64(predicate)) > bitDepth:
case uint64(bits.Len64(predicate)) > bitDepth:
fallthrough
case predicate == (1<<bitDepth)-1 && allowEquality:
// This query matches all possible values.
@ -1567,7 +1567,7 @@ func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint, predicate
case predicate == (1<<bitDepth)-1 && !allowEquality:
// This query matches everything that is not (1<<bitDepth)-1.
matches := NewRow()
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
row, err := f.row(tx, uint64(bsiOffsetBit+i))
if err != nil {
return nil, err
@ -1602,7 +1602,7 @@ func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint, predicate
return matched, nil
}
func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality bool) (*Row, error) {
func (f *fragment) rangeGT(tx Tx, bitDepth uint64, predicate int64, allowEquality bool) (*Row, error) {
if predicate == -1 && !allowEquality {
predicate, allowEquality = 0, true
}
@ -1644,7 +1644,7 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality
}
}
func (f *fragment) rangeGTUnsigned(tx Tx, filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
func (f *fragment) rangeGTUnsigned(tx Tx, filter *Row, bitDepth uint64, predicate uint64, allowEquality bool) (*Row, error) {
prep:
switch {
case predicate == 0 && allowEquality:
@ -1653,7 +1653,7 @@ prep:
case predicate == 0 && !allowEquality:
// This query matches everything that is not 0.
matches := NewRow()
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
row, err := f.row(tx, uint64(bsiOffsetBit+i))
if err != nil {
return nil, err
@ -1661,7 +1661,7 @@ prep:
matches = matches.Union(filter.Intersect(row))
}
return matches, nil
case !allowEquality && uint(bits.Len64(predicate)) > bitDepth:
case !allowEquality && uint64(bits.Len64(predicate)) > bitDepth:
// The predicate is bigger than the BSI width, so nothing can be bigger.
return NewRow(), nil
case allowEquality:
@ -1700,7 +1700,7 @@ func (f *fragment) notNull(tx Tx) (*Row, error) {
}
// rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax.
func (f *fragment) rangeBetween(tx Tx, bitDepth uint, predicateMin, predicateMax int64) (*Row, error) {
func (f *fragment) rangeBetween(tx Tx, bitDepth uint64, predicateMin, predicateMax int64) (*Row, error) {
b, err := f.row(tx, bsiExistsBit)
if err != nil {
return nil, err
@ -1749,7 +1749,7 @@ func (f *fragment) rangeBetween(tx Tx, bitDepth uint, predicateMin, predicateMax
}
// rangeBetweenUnsigned returns BSI columns for a range of values. Disregards the sign bit.
func (f *fragment) rangeBetweenUnsigned(tx Tx, filter *Row, bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
func (f *fragment) rangeBetweenUnsigned(tx Tx, filter *Row, bitDepth uint64, predicateMin, predicateMax uint64) (*Row, error) {
switch {
case predicateMax > (1<<bitDepth)-1:
// The upper bound cannot be violated.
@ -1781,11 +1781,11 @@ func (f *fragment) rangeBetweenUnsigned(tx Tx, filter *Row, bitDepth uint, predi
predicateMax &^= equalMask
var err error
remaining, err = f.rangeGTUnsigned(tx, remaining, uint(diffLen), predicateMin, true)
remaining, err = f.rangeGTUnsigned(tx, remaining, uint64(diffLen), predicateMin, true)
if err != nil {
return nil, err
}
remaining, err = f.rangeLTUnsigned(tx, remaining, uint(diffLen), predicateMax, true)
remaining, err = f.rangeLTUnsigned(tx, remaining, uint64(diffLen), predicateMax, true)
if err != nil {
return nil, err
}
@ -2628,7 +2628,7 @@ func (f *fragment) bulkImportMutex(tx Tx, rowIDs, columnIDs []uint64) error {
return errors.Wrap(f.importPositions(tx, toSet, toClear, rowSet), "importing positions")
}
func (f *fragment) importValueSmallWrite(tx Tx, columnIDs []uint64, values []int64, bitDepth uint, clear bool) error {
func (f *fragment) importValueSmallWrite(tx Tx, columnIDs []uint64, values []int64, bitDepth uint64, clear bool) error {
// TODO figure out how to avoid re-allocating these each time. Probably
// possible to store them on the fragment with a capacity based on
// MaxOpN. For now, we know that the total number of bits to be
@ -2663,7 +2663,7 @@ func (f *fragment) importValueSmallWrite(tx Tx, columnIDs []uint64, values []int
return err
}
rowSet := make(map[uint64]struct{}, bitDepth+1)
for i := uint(0); i < bitDepth+1; i++ {
for i := uint64(0); i < bitDepth+1; i++ {
rowSet[uint64(i)] = struct{}{}
}
err := f.importPositions(tx, toSet, toClear, rowSet)
@ -2680,7 +2680,7 @@ func (f *fragment) importValueSmallWrite(tx Tx, columnIDs []uint64, values []int
}
// importValue bulk imports a set of range-encoded values.
func (f *fragment) importValue(tx Tx, columnIDs []uint64, values []int64, bitDepth uint, clear bool) error {
func (f *fragment) importValue(tx Tx, columnIDs []uint64, values []int64, bitDepth uint64, clear bool) error {
f.mu.Lock()
defer f.mu.Unlock()
@ -3282,7 +3282,7 @@ func (f *fragment) blockToRoaringData(block int) ([]byte, error) {
// upgradeRoaringBSIv2 upgrades a fragment that contains old BSI formatting
// to a new BSI format (v2). The new format moves the "exists" bit to the
// beginning & adds a negative sign bit.
func upgradeRoaringBSIv2(f *fragment, bitDepth uint) (string, error) {
func upgradeRoaringBSIv2(f *fragment, bitDepth uint64) (string, error) {
// If flag set, already upgraded. Exit.
if f.storage.Flags&roaringFlagBSIv2 == 1 {
return "", nil

16
fragment_internal_test.go
View file

@ -456,7 +456,7 @@ func TestFragment_SetValue(t *testing.T) {
})
t.Run("QuickCheck", func(t *testing.T) {
if err := quick.Check(func(bitDepth uint, bitN uint64, values []uint64) bool {
if err := quick.Check(func(bitDepth uint64, bitN uint64, values []uint64) bool {
// Limit bit depth & maximum values.
bitDepth = (bitDepth % 62) + 1
bitN = (bitN % 99) + 1
@ -988,7 +988,7 @@ func TestFragment_Range(t *testing.T) {
// benchmarkSetValues is a helper function to explore, very roughly, the cost
// of setting values.
func benchmarkSetValues(b *testing.B, tx Tx, bitDepth uint, f *fragment, cfunc func(uint64) uint64) {
func benchmarkSetValues(b *testing.B, tx Tx, bitDepth uint64, f *fragment, cfunc func(uint64) uint64) {
column := uint64(0)
for i := 0; i < b.N; i++ {
// We're not checking the error because this is a benchmark.
@ -1000,7 +1000,7 @@ func benchmarkSetValues(b *testing.B, tx Tx, bitDepth uint, f *fragment, cfunc f
// Benchmark performance of setValue for BSI ranges.
func BenchmarkFragment_SetValue(b *testing.B) {
depths := []uint{4, 8, 16}
depths := []uint64{4, 8, 16}
for _, bitDepth := range depths {
name := fmt.Sprintf("Depth%d", bitDepth)
f, idx, tx := mustOpenFragment(b, "i", "f", viewBSIGroupPrefix+"foo", 0, "none")
@ -1021,7 +1021,7 @@ func BenchmarkFragment_SetValue(b *testing.B) {
// benchmarkImportValues is a helper function to explore, very roughly, the cost
// of setting values using the special setter used for imports.
func benchmarkImportValues(b *testing.B, tx Tx, bitDepth uint, f *fragment, cfunc func(uint64) uint64) {
func benchmarkImportValues(b *testing.B, tx Tx, bitDepth uint64, f *fragment, cfunc func(uint64) uint64) {
column := uint64(0)
b.StopTimer()
columns := make([]uint64, b.N)
@ -1040,7 +1040,7 @@ func benchmarkImportValues(b *testing.B, tx Tx, bitDepth uint, f *fragment, cfun
// Benchmark performance of setValue for BSI ranges.
func BenchmarkFragment_ImportValue(b *testing.B) {
depths := []uint{4, 8, 16}
depths := []uint64{4, 8, 16}
for _, bitDepth := range depths {
name := fmt.Sprintf("Depth%d", bitDepth)
f, idx, tx := mustOpenBSIFragment(b, "i", "f", viewBSIGroupPrefix+"foo", 0)
@ -4405,7 +4405,7 @@ func TestFragmentPositionsForValue(t *testing.T) {
tests := []struct {
columnID uint64
bitDepth uint
bitDepth uint64
value int64
clear bool
toSet []uint64
@ -5260,7 +5260,7 @@ func TestImportMultipleValues(t *testing.T) {
vals []int64
checkCols []uint64
checkVals []int64
depth uint
depth uint64
}{
{
cols: []uint64{0, 0},
@ -5312,7 +5312,7 @@ func TestImportValueRowCache(t *testing.T) {
cols []uint64
vals []int64
checkCols []uint64
depth uint
depth uint64
}
tests := []struct {
tc1 testCase

8
handler.go
View file

@ -226,11 +226,11 @@ type ImportRequest struct {
// ValidateWithTimestamp ensures that the payload of the request is valid.
func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
if ir.IndexCreatedAt != 0 && ir.FieldCreatedAt != 0 {
if ir.IndexCreatedAt != indexCreatedAt || ir.FieldCreatedAt != fieldCreatedAt {
return ErrPreconditionFailed
}
if (ir.IndexCreatedAt != 0 && ir.IndexCreatedAt != indexCreatedAt) ||
(ir.FieldCreatedAt != 0 && ir.FieldCreatedAt != fieldCreatedAt) {
return ErrPreconditionFailed
}
return nil
}

2
holder.go
View file

@ -235,7 +235,7 @@ func DefaultHolderConfig() *HolderConfig {
OpenIDAllocator: func(string) (*idAllocator, error) { return &idAllocator{}, nil },
TranslationSyncer: NopTranslationSyncer,
Serializer: NopSerializer,
Schemator: disco.NopSchemator,
Schemator: disco.InMemSchemator,
CacheFlushInterval: defaultCacheFlushInterval,
StatsClient: stats.NopStatsClient,
NewAttrStore: newNopAttrStore,

4
index.go
View file

@ -102,7 +102,7 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
holder: holder,
trackExistence: true,
schemator: disco.NopSchemator,
schemator: disco.InMemSchemator,
serializer: NopSerializer,
translateStores: make(map[int]TranslateStore),
@ -618,7 +618,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field
cfm := &CreateFieldMessage{
Index: i.name,
Field: name,
CreatedAt: 0,
CreatedAt: timestamp(),
Meta: fo,
}

2
mmap_test.go
View file

@ -30,7 +30,7 @@ type cv struct {
}
func forceSnapshotsCheckMapping(t *testing.T) {
depth := uint(6)
depth := uint64(6)
f, idx, tx := mustOpenBSIFragment(t, "i", "f", viewStandard, 0)
tx.Rollback()
f.Logger = logger.NewLogfLogger(t)

7
pilosa.go
View file

@ -20,6 +20,7 @@ import (
"regexp"
"time"
"github.com/pilosa/pilosa/v2/disco"
pnet "github.com/pilosa/pilosa/v2/net"
"github.com/pilosa/pilosa/v2/storage"
"github.com/pkg/errors"
@ -30,7 +31,7 @@ var (
ErrHostRequired = errors.New("host required")
ErrIndexRequired = errors.New("index required")
ErrIndexExists = errors.New("index already exists")
ErrIndexExists = disco.ErrIndexExists
ErrIndexNotFound = errors.New("index not found")
ErrForeignIndexNotFound = errors.New("foreign index not found")
@ -38,7 +39,7 @@ var (
// ErrFieldRequired is returned when no field is specified.
ErrFieldRequired = errors.New("field required")
ErrColumnRequired = errors.New("column required")
ErrFieldExists = errors.New("field already exists")
ErrFieldExists = disco.ErrFieldExists
ErrFieldNotFound = errors.New("field not found")
ErrBSIGroupNotFound = errors.New("bsigroup not found")
@ -54,7 +55,7 @@ var (
ErrDecimalOutOfRange = errors.New("decimal value out of range")
ErrViewRequired = errors.New("view required")
ErrViewExists = errors.New("view already exists")
ErrViewExists = disco.ErrViewExists
ErrInvalidView = errors.New("invalid view")
ErrInvalidCacheType = errors.New("invalid cache type")

33
server/handler_test.go
View file

@ -231,10 +231,28 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
body := strings.TrimSpace(w.Body.String())
target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth)
if body != target {
t.Fatalf("\n%s\n!=\n%s", target, body)
var bodySchema pilosa.Schema
if err := json.Unmarshal(w.Body.Bytes(),
&bodySchema); err != nil {
t.Fatalf("unexpected unmarshalling error: %v", err)
}
// DO NOT COMPARE `CreatedAt` - reset to 0
for _, i := range bodySchema.Indexes {
i.CreatedAt = 0
for _, f := range i.Fields {
f.CreatedAt = 0
}
}
//
var targetSchema pilosa.Schema
if err := json.Unmarshal([]byte(fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth)),
&targetSchema); err != nil {
t.Fatalf("unexpected unmarshalling error: %v", err)
}
if !reflect.DeepEqual(targetSchema, bodySchema) {
t.Fatalf("target: %+v\nbody: %+v\n", targetSchema, bodySchema)
}
})
@ -313,11 +331,6 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("getting schema: %v", err)
}
err = cmd.API.ApplySchema(context.Background(), &pilosa.Schema{Indexes: indexInfo}, false)
if err != nil {
t.Fatalf("applying schema: %v", err)
}
idx := indexInfo[0]
fld := indexInfo[0].Fields[0]
msg := pilosa.ImportRequest{
@ -357,7 +370,7 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, httpReq)
if w.Code != 412 {
t.Fatal("expected: Precondition Failed, got:" + w.Body.String())
t.Fatalf("expected: Precondition Failed, got: %d", w.Code)
}
})

10
view.go
View file

@ -494,7 +494,7 @@ func (v *view) clearBit(txOrig Tx, rowID, columnID uint64) (changed bool, err er
}
// value uses a column of bits to read a multi-bit value.
func (v *view) value(txOrig Tx, columnID uint64, bitDepth uint) (value int64, exists bool, err error) {
func (v *view) value(txOrig Tx, columnID uint64, bitDepth uint64) (value int64, exists bool, err error) {
shard := columnID / ShardWidth
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
@ -511,7 +511,7 @@ func (v *view) value(txOrig Tx, columnID uint64, bitDepth uint) (value int64, ex
}
// setValue uses a column of bits to set a multi-bit value.
func (v *view) setValue(txOrig Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) {
func (v *view) setValue(txOrig Tx, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) {
shard := columnID / ShardWidth
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
@ -534,7 +534,7 @@ func (v *view) setValue(txOrig Tx, columnID uint64, bitDepth uint, value int64)
}
// clearValue removes a specific value assigned to columnID
func (v *view) clearValue(txOrig Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) {
func (v *view) clearValue(txOrig Tx, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) {
shard := columnID / ShardWidth
frag := v.Fragment(shard)
if frag == nil {
@ -556,7 +556,7 @@ func (v *view) clearValue(txOrig Tx, columnID uint64, bitDepth uint, value int64
}
// rangeOp returns rows with a field value encoding matching the predicate.
func (v *view) rangeOp(qcx *Qcx, op pql.Token, bitDepth uint, predicate int64) (_ *Row, err0 error) {
func (v *view) rangeOp(qcx *Qcx, op pql.Token, bitDepth uint64, predicate int64) (_ *Row, err0 error) {
r := NewRow()
for _, frag := range v.allFragments() {
@ -576,7 +576,7 @@ func (v *view) rangeOp(qcx *Qcx, op pql.Token, bitDepth uint, predicate int64) (
}
// upgradeViewBSIv2 upgrades the fragments of v. Returns ok true if any fragment upgraded.
func upgradeViewBSIv2(v *view, bitDepth uint) (ok bool, _ error) {
func upgradeViewBSIv2(v *view, bitDepth uint64) (ok bool, _ error) {
// If reading from an old formatted BSI roaring bitmap, upgrade and reload.
for _, frag := range v.allFragments() {
if frag.storage.Flags&roaringFlagBSIv2 == 1 {