scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #847 from travisturner/field-range-between

Browse source 
Implements BETWEEN for Range queries.
This commit is contained in:
Travis Turner 2017-09-27 11:14:31 -05:00 • committed by GitHub
commit 917f41e8eb
14 changed files with 457 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
executor.go
View file

@ -715,28 +715,65 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
fieldName, cond = k, vv
}
// Only support integers for now.
value, ok := cond.Value.(int64)
if !ok {
return nil, errors.New("Range(): conditions only support integer values")
}
if cond.Op == pql.BETWEEN {
// Find field.
field := f.Field(fieldName)
if field == nil {
return nil, ErrFieldNotFound
} else if value < field.Min || value > field.Max {
return NewBitmap(), nil
}
predicates, err := cond.IntSliceValue()
if err != nil {
return nil, err
}
// Retrieve fragment.
frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice)
if frag == nil {
return NewBitmap(), nil
}
// Only support two integers for the between operation.
if len(predicates) != 2 {
return nil, errors.New("Range(): BETWEEN condition requires exactly two integer values")
}
f.Stats.Count("range:field", 1, 1.0)
return frag.FieldRange(cond.Op, field.BitDepth(), uint64(value-field.Min))
// Find field.
field := f.Field(fieldName)
if field == nil {
return nil, ErrFieldNotFound
}
baseValueMin, baseValueMax, outOfRange := field.BaseValueBetween(predicates[0], predicates[1])
if outOfRange {
return NewBitmap(), nil
}
// Retrieve fragment.
frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice)
if frag == nil {
return NewBitmap(), nil
}
return frag.FieldRangeBetween(field.BitDepth(), baseValueMin, baseValueMax)
} else {
// Only support integers for now.
value, ok := cond.Value.(int64)
if !ok {
return nil, errors.New("Range(): conditions only support integer values")
}
// Find field.
field := f.Field(fieldName)
if field == nil {
return nil, ErrFieldNotFound
}
baseValue, outOfRange := field.BaseValue(cond.Op, value)
if outOfRange {
return NewBitmap(), nil
}
// Retrieve fragment.
frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice)
if frag == nil {
return NewBitmap(), nil
}
f.Stats.Count("range:field", 1, 1.0)
return frag.FieldRange(cond.Op, field.BitDepth(), baseValue)
}
}
// executeUnionSlice executes a union() call for a local slice.

49
fragment.go
View file

@ -641,7 +641,10 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b
}
// If bit is set then add columns for set bits to exclude.
keep = keep.Union(b.Difference(row))
// Don't bother to compute this on the final iteration.
if i > 0 {
keep = keep.Union(b.Difference(row))
}
}
return b, nil
@ -673,7 +676,49 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b
}
// If bit is unset then add columns with set bit to keep.
keep = keep.Union(b.Intersect(row))
// Don't bother to compute this on the final iteration.
if i > 0 {
keep = keep.Union(b.Intersect(row))
}
}
return b, nil
}
func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Bitmap, error) {
b := f.Row(uint64(bitDepth))
keep1 := NewBitmap() // GTE
keep2 := NewBitmap() // LTE
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
bit1 := (predicateMin >> uint(i)) & 1
bit2 := (predicateMax >> uint(i)) & 1
// GTE predicateMin
// If bit is set then remove all unset columns not already kept.
if bit1 == 1 {
b = b.Difference(b.Difference(row).Difference(keep1))
} else {
// If bit is unset then add columns with set bit to keep.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep1 = keep1.Union(b.Intersect(row))
}
}
// LTE predicateMin
// If bit is zero then remove all set columns not in excluded bitmap.
if bit2 == 0 {
b = b.Difference(row.Difference(keep2))
} else {
// If bit is set then add columns for set bits to exclude.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep2 = keep2.Union(b.Difference(row))
}
}
}
return b, nil

48
fragment_test.go
View file

@ -378,6 +378,54 @@ func TestFragment_FieldRange(t *testing.T) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
})
t.Run("BETWEEN", func(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil {
t.Fatal(err)
}
// Query for fields greater than (ending with unset bit).
if b, err := f.FieldRangeBetween(bitDepth, 300, 2817); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
// Query for fields greater than (ending with set bit).
if b, err := f.FieldRangeBetween(bitDepth, 301, 2817); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
// Query for fields greater than or equal to (ending with unset bit).
if b, err := f.FieldRangeBetween(bitDepth, 301, 2816); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
// Query for fields greater than or equal to (ending with set bit).
if b, err := f.FieldRangeBetween(bitDepth, 300, 2816); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
})
}
// Ensure a fragment can snapshot correctly.

81
frame.go
View file

@ -746,12 +746,37 @@ func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap,
return nil, nil
}
// Adjust predicate to range.
baseValue := uint64(predicate - field.Min)
baseValue, outOfRange := field.BaseValue(op, predicate)
if outOfRange {
return NewBitmap(), nil
}
return view.FieldRange(op, field.BitDepth(), baseValue)
}
func (f *Frame) FieldRangeBetween(name string, predicateMin, predicateMax int64) (*Bitmap, error) {
// Retrieve and validate field.
field := f.Field(name)
if field == nil {
return nil, ErrFieldNotFound
} else if predicateMin > predicateMax {
return nil, ErrInvalidBetweenValue
}
// Retrieve field's view.
view := f.View(ViewFieldPrefix + name)
if view == nil {
return nil, nil
}
baseValueMin, baseValueMax, outOfRange := field.BaseValueBetween(predicateMin, predicateMax)
if outOfRange {
return NewBitmap(), nil
}
return view.FieldRangeBetween(field.BitDepth(), baseValueMin, baseValueMax)
}
// Import bulk imports data.
func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error {
// Determine quantum if timestamps are set.
@ -999,6 +1024,58 @@ func (f *Field) BitDepth() uint {
return 63
}
// BaseValue adjusts the value to align with the range for Field for a certain
// operation type.
// TODO: there is an edge case for GT and LT where this returns a baseValue
// that does not fully encompass the range.
// ex: Field.Min = 0, Field.Max = 1023
// BaseValue(LT, 2000) returns 1023, which will perform "LT 1023" and effectively
// exclude any columns with value = 1023.
// Note that in this case (because the range uses the full BitDepth 0 to 1023),
// we can't simply return 1024.
// In order to make this work, we effectively need to change the operator to LTE.
func (f *Field) BaseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) {
if op == pql.GT || op == pql.GTE {
if value > f.Max {
return baseValue, true
} else if value > f.Min {
baseValue = uint64(value - f.Min)
}
} else if op == pql.LT || op == pql.LTE {
if value < f.Min {
return baseValue, true
} else if value > f.Max {
baseValue = uint64(f.Max - f.Min)
} else {
baseValue = uint64(value - f.Min)
}
} else if op == pql.EQ {
if value < f.Min || value > f.Max {
return baseValue, true
}
baseValue = uint64(value - f.Min)
}
return baseValue, false
}
// BaseValueBetween adjusts the min/max value to align with the range for Field.
func (f *Field) BaseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) {
if max < f.Min || min > f.Max {
return baseValueMin, baseValueMax, true
}
// Adjust min/max to range.
if min > f.Min {
baseValueMin = uint64(min - f.Min)
}
// Make sure the high value of the BETWEEN does not exceed BitDepth.
if max > f.Max {
baseValueMax = uint64(f.Max - f.Min)
} else if max > f.Min {
baseValueMax = uint64(max - f.Min)
}
return baseValueMin, baseValueMax, false
}
func ValidateField(f *Field) error {
if f.Name == "" {
return ErrFieldNameRequired

129
frame_test.go
View file

@ -16,9 +16,11 @@ package pilosa_test
import (
"io/ioutil"
"reflect"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
)
@ -340,3 +342,130 @@ func TestFrame_DeleteView(t *testing.T) {
t.Fatal("failed to create new view")
}
}
// Ensure a field can adjust to its baseValue.
func TestField_BaseValue(t *testing.T) {
f0 := &pilosa.Field{
Name: "f0",
Type: pilosa.FieldTypeInt,
Min: -100,
Max: 900,
}
f1 := &pilosa.Field{
Name: "f1",
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 1000,
}
f2 := &pilosa.Field{
Name: "f2",
Type: pilosa.FieldTypeInt,
Min: 100,
Max: 1100,
}
t.Run("Normal Condition", func(t *testing.T) {
for _, tt := range []struct {
f *pilosa.Field
op pql.Token
val int64
expBaseValue uint64
expOutOfRange bool
}{
// LT
{f0, pql.LT, 5, 105, false},
{f0, pql.LT, -8, 92, false},
{f0, pql.LT, -108, 0, true},
{f0, pql.LT, 1005, 1000, false},
{f0, pql.LT, 0, 100, false},
{f1, pql.LT, 5, 5, false},
{f1, pql.LT, -8, 0, true},
{f1, pql.LT, 1005, 1000, false},
{f1, pql.LT, 0, 0, false},
{f2, pql.LT, 5, 0, true},
{f2, pql.LT, -8, 0, true},
{f2, pql.LT, 105, 5, false},
{f2, pql.LT, 1105, 1000, false},
// GT
{f0, pql.GT, -105, 0, false},
{f0, pql.GT, 5, 105, false},
{f0, pql.GT, 905, 0, true},
{f0, pql.GT, 0, 100, false},
{f1, pql.GT, 5, 5, false},
{f1, pql.GT, -8, 0, false},
{f1, pql.GT, 1005, 0, true},
{f1, pql.GT, 0, 0, false},
{f2, pql.GT, 5, 0, false},
{f2, pql.GT, -8, 0, false},
{f2, pql.GT, 105, 5, false},
{f2, pql.GT, 1105, 0, true},
// EQ
{f0, pql.EQ, -105, 0, true},
{f0, pql.EQ, 5, 105, false},
{f0, pql.EQ, 905, 0, true},
{f0, pql.EQ, 0, 100, false},
{f1, pql.EQ, 5, 5, false},
{f1, pql.EQ, -8, 0, true},
{f1, pql.EQ, 1005, 0, true},
{f1, pql.EQ, 0, 0, false},
{f2, pql.EQ, 5, 0, true},
{f2, pql.EQ, -8, 0, true},
{f2, pql.EQ, 105, 5, false},
{f2, pql.EQ, 1105, 0, true},
} {
bv, oor := tt.f.BaseValue(tt.op, tt.val)
if oor != tt.expOutOfRange {
t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor)
} else if !reflect.DeepEqual(bv, tt.expBaseValue) {
t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv)
}
}
})
t.Run("Betwween Condition", func(t *testing.T) {
for _, tt := range []struct {
f *pilosa.Field
predMin int64
predMax int64
expBaseValueMin uint64
expBaseValueMax uint64
expOutOfRange bool
}{
{f0, -205, -105, 0, 0, true},
{f0, -105, 80, 0, 180, false},
{f0, 5, 20, 105, 120, false},
{f0, 20, 1005, 120, 1000, false},
{f0, 1005, 2000, 0, 0, true},
{f1, -105, -5, 0, 0, true},
{f1, -5, 20, 0, 20, false},
{f1, 5, 20, 5, 20, false},
{f1, 20, 1005, 20, 1000, false},
{f1, 1005, 2000, 0, 0, true},
{f2, 5, 95, 0, 0, true},
{f2, 95, 120, 0, 20, false},
{f2, 105, 120, 5, 20, false},
{f2, 120, 1105, 20, 1000, false},
{f2, 1105, 2000, 0, 0, true},
} {
min, max, oor := tt.f.BaseValueBetween(tt.predMin, tt.predMax)
if oor != tt.expOutOfRange {
t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor)
} else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) {
t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max)
}
}
})
}

1
pilosa.go
View file

@ -60,6 +60,7 @@ var (
ErrFieldValueTooLow = errors.New("field value too low")
ErrFieldValueTooHigh = errors.New("field value too high")
ErrInvalidRangeOperation = errors.New("invalid range operation")
ErrInvalidBetweenValue = errors.New("invalid value for between operation")
ErrInvalidView = errors.New("invalid view")
ErrInvalidCacheType = errors.New("invalid cache type")

25
pql/ast.go
View file

@ -220,6 +220,31 @@ func (cond *Condition) String() string {
return fmt.Sprintf("%s %s", cond.Op.String(), FormatValue(cond.Value))
}
// IntSliceValue reads cond.Value as a slice of uint64.
// If the value is a slice of uint64 it will convert
// it to []int64. Otherwise, if it is not a []int64 it will return an error.
func (cond *Condition) IntSliceValue() ([]int64, error) {
val := cond.Value
switch tval := val.(type) {
case []interface{}:
ret := make([]int64, len(tval))
for i, v := range tval {
switch tv := v.(type) {
case int64:
ret[i] = tv
case uint64:
ret[i] = int64(tv)
default:
return nil, fmt.Errorf("unexpected value type %T in IntSliceValue, val %v", tv, tv)
}
}
return ret, nil
default:
return nil, fmt.Errorf("unexpected type %T in IntSliceValue, val %v", tval, tval)
}
}
func FormatValue(v interface{}) string {
switch v := v.(type) {
case string:

26
pql/ast_test.go
View file

@ -15,6 +15,7 @@
package pql_test
import (
"reflect"
"testing"
"github.com/pilosa/pilosa/pql"
@ -30,6 +31,31 @@ func TestCall_String(t *testing.T) {
})
}
// Ensure condition can handle values for BETWEEN operator.
func TestCondition_Value(t *testing.T) {
t.Run("Between Values", func(t *testing.T) {
for _, tt := range []struct {
val []interface{}
exp []int64
}{
{[]interface{}{int64(4), int64(8)}, []int64{4, 8}},
{[]interface{}{uint64(4), uint64(8)}, []int64{4, 8}},
{[]interface{}{uint64(1), uint64(2), uint64(3)}, []int64{1, 2, 3}},
} {
c := &pql.Condition{
Op: pql.BETWEEN,
Value: tt.val,
}
v, err := c.IntSliceValue()
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(v, tt.exp) {
t.Fatalf("invalid between values. expected: %v, got %v", tt.exp, v)
}
}
})
}
// Ensure call can be converted into a string.
func TestCall_SupportsInverse(t *testing.T) {
t.Run("Bitmap", func(t *testing.T) {

2
pql/parser.go
View file

@ -165,7 +165,7 @@ func (p *Parser) parseArgs() (map[string]interface{}, error) {
var op Token
switch tok, pos, lit := p.scanIgnoreWhitespace(); tok {
case ASSIGN:
case EQ, LT, LTE, GT, GTE:
case EQ, LT, LTE, GT, GTE, BETWEEN:
op = tok
default:
return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit)

3
pql/parser_test.go
View file

@ -172,7 +172,7 @@ func TestParser_Parse(t *testing.T) {
// Parse with condition arguments.
t.Run("WithCondition", func(t *testing.T) {
q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100)`)
q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8])`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
@ -182,6 +182,7 @@ func TestParser_Parse(t *testing.T) {
"key": "foo",
"x": &pql.Condition{Op: pql.EQ, Value: 12.25},
"y": &pql.Condition{Op: pql.GTE, Value: int64(100)},
"z": &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}},
},
},
) {

5
pql/scanner.go
View file

@ -73,8 +73,11 @@ func (s *Scanner) Scan() (tok Token, pos Pos, lit string) {
s.unread()
return LT, pos, string(ch)
case '>':
if next := s.read(); next == '=' {
next := s.read()
if next == '=' {
return GTE, pos, ">="
} else if next == '<' {
return BETWEEN, pos, "><"
}
s.unread()
return GT, pos, string(ch)

1
pql/scanner_test.go
View file

@ -42,6 +42,7 @@ func TestScanner_Scan(t *testing.T) {
{name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`},
{name: "GT", s: `>`, tok: pql.GT, lit: `>`},
{name: "GTE", s: `>=`, tok: pql.GTE, lit: `>=`},
{name: "BETWEEN", s: `><`, tok: pql.BETWEEN, lit: `><`},
{name: "COMMA", s: `,`, tok: pql.COMMA, lit: `,`},
{name: "LPAREN", s: `(`, tok: pql.LPAREN, lit: `(`},
{name: "RPAREN", s: `)`, tok: pql.RPAREN, lit: `)`},

46
pql/token.go
View file

@ -37,17 +37,18 @@ const (
ALL
keyword_end
ASSIGN // =
EQ // ==
LT // <
LTE // <=
GT // >
GTE // >=
COMMA // ,
LPAREN // (
RPAREN // )
LBRACK // (
RBRACK // )
ASSIGN // =
EQ // ==
LT // <
LTE // <=
GT // >
GTE // >=
BETWEEN // ><
COMMA // ,
LPAREN // (
RPAREN // )
LBRACK // (
RBRACK // )
)
var tokens = [...]string{
@ -61,17 +62,18 @@ var tokens = [...]string{
ALL: "ALL",
ASSIGN: "=",
EQ: "==",
LT: "<",
LTE: "<=",
GT: ">",
GTE: ">=",
COMMA: ",",
LPAREN: "(",
RPAREN: ")",
LBRACK: "(",
RBRACK: ")",
ASSIGN: "=",
EQ: "==",
LT: "<",
LTE: "<=",
GT: ">",
GTE: ">=",
BETWEEN: "><",
COMMA: ",",
LPAREN: "(",
RPAREN: ")",
LBRACK: "(",
RBRACK: ")",
}
var keywords map[string]Token

14
view.go
View file

@ -329,6 +329,20 @@ func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitma
return bm, nil
}
// FieldRangeBetween returns bitmaps with a field value encoding matching any
// value between predicateMin and predicateMax.
func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Bitmap, error) {
bm := NewBitmap()
for _, frag := range v.Fragments() {
other, err := frag.FieldRangeBetween(bitDepth, predicateMin, predicateMax)
if err != nil {
return nil, err
}
bm = bm.Union(other)
}
return bm, nil
}
// IsInverseView returns true if the view is used for storing an inverted representation.
func IsInverseView(name string) bool {
return strings.HasPrefix(name, ViewInverse)