scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-11 15:21:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #874 from travisturner/query-field-not-null

Browse source 
FieldNotNull for more efficient BETWEEN queries
This commit is contained in:
Travis Turner 2017-10-06 15:14:55 -05:00 • committed by GitHub
commit 71c97ab44f
10 changed files with 138 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
executor.go
View file

@ -715,7 +715,30 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
fieldName, cond = k, vv
}
if cond.Op == pql.BETWEEN {
// EQ null (not implemented: flip frag.FieldNotNull with max ColumnID)
// NEQ null frag.FieldNotNull()
// BETWEEN a,b(in) BETWEEN/frag.FieldRangeBetween()
// BETWEEN a,b(out) BETWEEN/frag.FieldNotNull()
// EQ <int> frag.FieldRange
// NEQ <int> frag.FieldRange
// Handle `!= null`.
if cond.Op == pql.NEQ && cond.Value == nil {
// Find field.
field := f.Field(fieldName)
if field == nil {
return nil, ErrFieldNotFound
}
// Retrieve fragment.
frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice)
if frag == nil {
return NewBitmap(), nil
}
return frag.FieldNotNull(field.BitDepth())
} else if cond.Op == pql.BETWEEN {
predicates, err := cond.IntSliceValue()
if err != nil {
@ -727,6 +750,10 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
return nil, errors.New("Range(): BETWEEN condition requires exactly two integer values")
}
// The reason we don't just call:
// return f.FieldRangeBetween(fieldName, predicates[0], predicates[1])
// here is because we need the call to be slice-specific.
// Find field.
field := f.Field(fieldName)
if field == nil {
@ -744,6 +771,12 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
return NewBitmap(), nil
}
// If the query is asking for the entire valid range, just return
// the not-null bitmap for the field.
if predicates[0] <= field.Min && predicates[1] >= field.Max {
return frag.FieldNotNull(field.BitDepth())
}
return frag.FieldRangeBetween(field.BitDepth(), baseValueMin, baseValueMax)
} else {
@ -761,7 +794,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
}
baseValue, outOfRange := field.BaseValue(cond.Op, value)
if outOfRange {
if outOfRange && cond.Op != pql.NEQ {
return NewBitmap(), nil
}
@ -771,6 +804,11 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
return NewBitmap(), nil
}
// outOfRange for NEQ should return all not-null.
if outOfRange && cond.Op == pql.NEQ {
return frag.FieldNotNull(field.BitDepth())
}
f.Stats.Count("range:field", 1, 1.0)
return frag.FieldRange(cond.Op, field.BitDepth(), baseValue)
}

39
executor_test.go
View file

@ -764,6 +764,28 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
}
})
t.Run("NEQ", func(t *testing.T) {
// NEQ null
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != null)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Bitmap).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ <int>
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo != 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Bitmap).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ -<int>
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != -20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Bitmap).Bits()) {
//t.Fatalf("unexpected result: %s", spew.Sdump(result))
t.Fatalf("unexpected result: %s", result[0].(*pilosa.Bitmap).Bits())
}
})
t.Run("LT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil {
t.Fatal(err)
@ -796,6 +818,23 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
}
})
t.Run("BETWEEN", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [1, 1000])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Bitmap).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
// Ensure that the FieldNotNull code path gets run.
t.Run("FieldNotNull", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [0, 1000])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Bitmap).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("BelowMin", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil {
t.Fatal(err)

23
fragment.go
View file

@ -619,6 +619,8 @@ func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*B
switch op {
case pql.EQ:
return f.fieldRangeEQ(bitDepth, predicate)
case pql.NEQ:
return f.fieldRangeNEQ(bitDepth, predicate)
case pql.LT, pql.LTE:
return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE)
case pql.GT, pql.GTE:
@ -647,6 +649,22 @@ func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Bitmap, error
return b, nil
}
func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Bitmap, error) {
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
// Get the equal bitmap.
eq, err := f.fieldRangeEQ(bitDepth, predicate)
if err != nil {
return nil, err
}
// Not-null minus the equal bitmap.
b = b.Difference(eq)
return b, nil
}
func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Bitmap, error) {
keep := NewBitmap()
@ -730,6 +748,11 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b
return b, nil
}
// FieldNotNull returns the not-null row (stored at bitDepth).
func (f *Fragment) FieldNotNull(bitDepth uint) (*Bitmap, error) {
return f.Row(uint64(bitDepth)), nil
}
func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Bitmap, error) {
b := f.Row(uint64(bitDepth))
keep1 := NewBitmap() // GTE

23
fragment_test.go
View file

@ -283,6 +283,29 @@ func TestFragment_FieldRange(t *testing.T) {
}
})
t.Run("NEQ", func(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
t.Fatal(err)
}
// Query for inequality.
if b, err := f.FieldRange(pql.NEQ, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
})
t.Run("LT", func(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()

2
frame.go
View file

@ -1119,7 +1119,7 @@ func (f *Field) BaseValue(op pql.Token, value int64) (baseValue uint64, outOfRan
} else {
baseValue = uint64(value - f.Min)
}
} else if op == pql.EQ {
} else if op == pql.EQ || op == pql.NEQ {
if value < f.Min || value > f.Max {
return baseValue, true
}

2
pql/parser.go
View file

@ -165,7 +165,7 @@ func (p *Parser) parseArgs() (map[string]interface{}, error) {
var op Token
switch tok, pos, lit := p.scanIgnoreWhitespace(); tok {
case ASSIGN:
case EQ, LT, LTE, GT, GTE, BETWEEN:
case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN:
op = tok
default:
return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit)

3
pql/parser_test.go
View file

@ -172,7 +172,7 @@ func TestParser_Parse(t *testing.T) {
// Parse with condition arguments.
t.Run("WithCondition", func(t *testing.T) {
q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8])`)
q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
@ -183,6 +183,7 @@ func TestParser_Parse(t *testing.T) {
"x": &pql.Condition{Op: pql.EQ, Value: 12.25},
"y": &pql.Condition{Op: pql.GTE, Value: int64(100)},
"z": &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}},
"m": &pql.Condition{Op: pql.NEQ, Value: nil},
},
},
) {

6
pql/scanner.go
View file

@ -66,6 +66,12 @@ func (s *Scanner) Scan() (tok Token, pos Pos, lit string) {
}
s.unread()
return ASSIGN, pos, string(ch)
case '!':
if next := s.read(); next == '=' {
return NEQ, pos, "!="
}
s.unread()
return ASSIGN, pos, string(ch)
case '<':
if next := s.read(); next == '=' {
return LTE, pos, "<="

1
pql/scanner_test.go
View file

@ -38,6 +38,7 @@ func TestScanner_Scan(t *testing.T) {
{name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`},
{name: "EQ", s: `==`, tok: pql.EQ, lit: `==`},
{name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`},
{name: "LT", s: `<`, tok: pql.LT, lit: `<`},
{name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`},
{name: "GT", s: `>`, tok: pql.GT, lit: `>`},

2
pql/token.go
View file

@ -39,6 +39,7 @@ const (
ASSIGN // =
EQ // ==
NEQ // !=
LT // <
LTE // <=
GT // >
@ -64,6 +65,7 @@ var tokens = [...]string{
ASSIGN: "=",
EQ: "==",
NEQ: "!=",
LT: "<",
LTE: "<=",
GT: ">",