diff --git a/executor.go b/executor.go index aa5b69034..fd1e921d9 100644 --- a/executor.go +++ b/executor.go @@ -620,6 +620,11 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p // executeRangeSlice executes a range() call for a local slice. func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { + // Handle field ranges differently. + if c.HasConditionArg() { + return e.executeFieldRangeSlice(ctx, index, c, slice) + } + // Parse frame, use default if unset. frame, _ := c.Args["frame"].(string) if frame == "" { @@ -674,7 +679,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Parse end time. - endTimeStr, _ := c.Args["end"].(string) + endTimeStr, ok := c.Args["end"].(string) if !ok { return nil, errors.New("Range() end time required") } @@ -702,6 +707,64 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return bm, nil } +// executeFieldRangeSlice executes a range(field) call for a local slice. +func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { + // Parse frame, use default if unset. + frame, _ := c.Args["frame"].(string) + if frame == "" { + frame = DefaultFrame + } + f := e.Holder.Frame(index, frame) + if f == nil { + return nil, ErrFrameNotFound + } + + // Remove frame field. + args := pql.CopyArgs(c.Args) + delete(args, "frame") + + // Only one conditional field should remain. + if len(args) == 0 { + return nil, errors.New("Range(): condition required") + } else if len(args) > 1 { + return nil, errors.New("Range(): too many arguments") + } + + // Extract condition field. + var fieldName string + var cond *pql.Condition + for k, v := range args { + vv, ok := v.(*pql.Condition) + if !ok { + return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v) + } + fieldName, cond = k, vv + } + + // Only support integers for now. + value, ok := cond.Value.(int64) + if !ok { + return nil, errors.New("Range(): conditions only support integer values") + } + + // Find field. + field := f.Field(fieldName) + if field == nil { + return nil, ErrFieldNotFound + } else if value < field.Min || value > field.Max { + return NewBitmap(), nil + } + + // Retrieve fragment. + frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + if frag == nil { + return NewBitmap(), nil + } + + f.Stats.Count("range:field", 1, 1.0) + return frag.FieldRange(cond.Op, field.BitDepth(), uint64(value-field.Min)) +} + // executeUnionSlice executes a union() call for a local slice. func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { other := NewBitmap() diff --git a/executor_test.go b/executor_test.go index 265293aef..a7186a455 100644 --- a/executor_test.go +++ b/executor_test.go @@ -735,6 +735,119 @@ func TestExecutor_Execute_Range(t *testing.T) { }) } +// Ensure a Range(field) query can be executed. +func TestExecutor_Execute_FieldRange(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, + {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(frame=f, rowID=0, columnID=0) + SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + + SetFieldValue(frame=f, foo=20, bar=2000, columnID=50) + SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=10, columnID=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=20, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, foo=1000, columnID=0) + `), nil, nil); err != nil { + t.Fatal(err) + } + + t.Run("EQ", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("LT", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("LTE", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("GT", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("GTE", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("BelowMin", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("AboveMax", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("ErrFrameNotFound", func(t *testing.T) { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=bad_frame, foo >= 20)`), nil, nil); err != pilosa.ErrFrameNotFound { + t.Fatal(err) + } + }) + + t.Run("ErrFieldNotFound", func(t *testing.T) { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { + t.Fatal(err) + } + }) +} + // Ensure a remote query can return a bitmap. func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { c := test.NewCluster(2) diff --git a/fragment.go b/fragment.go index a517cbd57..06f451250 100644 --- a/fragment.go +++ b/fragment.go @@ -39,6 +39,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" ) @@ -567,14 +568,14 @@ func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, e return sum, count, nil } -func (f *Fragment) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) { +func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { switch op { - case RangeOpEQ: + case pql.EQ: return f.fieldRangeEQ(bitDepth, predicate) - case RangeOpLT, RangeOpLTE: - return f.fieldRangeLT(bitDepth, predicate, op == RangeOpLTE) - case RangeOpGT, RangeOpGTE: - return f.fieldRangeGT(bitDepth, predicate, op == RangeOpGTE) + case pql.LT, pql.LTE: + return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE) + case pql.GT, pql.GTE: + return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE) default: return nil, ErrInvalidRangeOperation } diff --git a/fragment_test.go b/fragment_test.go index 7dfdac8ae..941554db4 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -24,6 +24,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" ) @@ -275,7 +276,7 @@ func TestFragment_FieldRange(t *testing.T) { } // Query for equality. - if b, err := f.FieldRange(pilosa.RangeOpEQ, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) @@ -302,28 +303,28 @@ func TestFragment_FieldRange(t *testing.T) { } // Query for fields less than (ending with set bit). - if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 301); err != nil { + if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields less than (ending with unset bit). - if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields less than or equal to (ending with set bit). - if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 301); err != nil { + if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields less than or equal to (ending with unset bit). - if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) @@ -350,28 +351,28 @@ func TestFragment_FieldRange(t *testing.T) { } // Query for fields greater than (ending with unset bit). - if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields greater than (ending with set bit). - if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 301); err != nil { + if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields greater than or equal to (ending with unset bit). - if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields greater than or equal to (ending with set bit). - if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 301); err != nil { + if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) diff --git a/frame.go b/frame.go index 19c25eeab..9ca2d6109 100644 --- a/frame.go +++ b/frame.go @@ -27,6 +27,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" ) // Default frame settings. @@ -40,15 +41,6 @@ const ( DefaultCacheSize = 50000 ) -// List of operators for field range queries. -const ( - RangeOpEQ = "eq" - RangeOpLT = "lt" - RangeOpLTE = "lte" - RangeOpGT = "gt" - RangeOpGTE = "gte" -) - // Frame represents a container for views. type Frame struct { mu sync.Mutex @@ -663,7 +655,7 @@ func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err err return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil } -func (f *Frame) FieldRange(name, op string, predicate int64) (*Bitmap, error) { +func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap, error) { // Retrieve and validate field. field := f.Field(name) if field == nil { diff --git a/pql/ast.go b/pql/ast.go index 9555f9d90..48e5f1771 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -198,6 +198,16 @@ func (c *Call) IsInverse(rowLabel, columnLabel string) bool { return false } +// HasConditionArg returns true if any arg is a conditional. +func (c *Call) HasConditionArg() bool { + for _, v := range c.Args { + if _, ok := v.(*Condition); ok { + return true + } + } + return false +} + // Condition represents an operation & value. // When used in an argument map it represents a binary expression. type Condition struct { diff --git a/view.go b/view.go index 31a7a53c0..99ce3f7f9 100644 --- a/view.go +++ b/view.go @@ -25,6 +25,7 @@ import ( "sync" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" ) // View layout modes. @@ -314,7 +315,7 @@ func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err e } // FieldRange returns bitmaps with a field value encoding matching the predicate. -func (v *View) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) { +func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { bm := NewBitmap() for _, frag := range v.Fragments() { other, err := frag.FieldRange(op, bitDepth, predicate)