Merge pull request #791 from benbjohnson/pql-range

Add field Range() support to Executor.

executor.go

@@ -629,6 +629,11 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p
 
 // executeRangeSlice executes a range() call for a local slice.
 func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
+	// Handle field ranges differently.
+	if c.HasConditionArg() {
+		return e.executeFieldRangeSlice(ctx, index, c, slice)
+	}
+
 	// Parse frame, use default if unset.
 	frame, _ := c.Args["frame"].(string)
 	if frame == "" {
@@ -683,7 +688,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
 	}
 
 	// Parse end time.
-	endTimeStr, _ := c.Args["end"].(string)
+	endTimeStr, ok := c.Args["end"].(string)
 	if !ok {
 		return nil, errors.New("Range() end time required")
 	}
@@ -711,6 +716,64 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
 	return bm, nil
 }
 
+// executeFieldRangeSlice executes a range(field) call for a local slice.
+func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
+	// Parse frame, use default if unset.
+	frame, _ := c.Args["frame"].(string)
+	if frame == "" {
+		frame = DefaultFrame
+	}
+	f := e.Holder.Frame(index, frame)
+	if f == nil {
+		return nil, ErrFrameNotFound
+	}
+
+	// Remove frame field.
+	args := pql.CopyArgs(c.Args)
+	delete(args, "frame")
+
+	// Only one conditional field should remain.
+	if len(args) == 0 {
+		return nil, errors.New("Range(): condition required")
+	} else if len(args) > 1 {
+		return nil, errors.New("Range(): too many arguments")
+	}
+
+	// Extract condition field.
+	var fieldName string
+	var cond *pql.Condition
+	for k, v := range args {
+		vv, ok := v.(*pql.Condition)
+		if !ok {
+			return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v)
+		}
+		fieldName, cond = k, vv
+	}
+
+	// Only support integers for now.
+	value, ok := cond.Value.(int64)
+	if !ok {
+		return nil, errors.New("Range(): conditions only support integer values")
+	}
+
+	// Find field.
+	field := f.Field(fieldName)
+	if field == nil {
+		return nil, ErrFieldNotFound
+	} else if value < field.Min || value > field.Max {
+		return NewBitmap(), nil
+	}
+
+	// Retrieve fragment.
+	frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice)
+	if frag == nil {
+		return NewBitmap(), nil
+	}
+
+	f.Stats.Count("range:field", 1, 1.0)
+	return frag.FieldRange(cond.Op, field.BitDepth(), uint64(value-field.Min))
+}
+
 // executeUnionSlice executes a union() call for a local slice.
 func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
 	other := NewBitmap()

executor_test.go

@@ -773,6 +773,119 @@ func TestExecutor_Execute_Range(t *testing.T) {
 	})
 }
 
+// Ensure a Range(field) query can be executed.
+func TestExecutor_Execute_FieldRange(t *testing.T) {
+	hldr := test.MustOpenHolder()
+	defer hldr.Close()
+	e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
+
+	idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
+		RangeEnabled: true,
+		Fields: []*pilosa.Field{
+			{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
+			{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
+		},
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
+		RangeEnabled: true,
+		Fields: []*pilosa.Field{
+			{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
+		},
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := e.Execute(context.Background(), "i", test.MustParse(`
+		SetBit(frame=f, rowID=0, columnID=0)
+		SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`)
+
+		SetFieldValue(frame=f, foo=20, bar=2000, columnID=50)
+		SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`)
+		SetFieldValue(frame=f, foo=10, columnID=`+strconv.Itoa(SliceWidth+2)+`)
+		SetFieldValue(frame=f, foo=20, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
+		SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`)
+		SetFieldValue(frame=other, foo=1000, columnID=0)
+	`), nil, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("EQ", func(t *testing.T) {
+		if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) {
+			t.Fatalf("unexpected result: %s", spew.Sdump(result))
+		}
+	})
+
+	t.Run("LT", func(t *testing.T) {
+		if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Bitmap).Bits()) {
+			t.Fatalf("unexpected result: %s", spew.Sdump(result))
+		}
+	})
+
+	t.Run("LTE", func(t *testing.T) {
+		if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) {
+			t.Fatalf("unexpected result: %s", spew.Sdump(result))
+		}
+	})
+
+	t.Run("GT", func(t *testing.T) {
+		if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Bitmap).Bits()) {
+			t.Fatalf("unexpected result: %s", spew.Sdump(result))
+		}
+	})
+
+	t.Run("GTE", func(t *testing.T) {
+		if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) {
+			t.Fatalf("unexpected result: %s", spew.Sdump(result))
+		}
+	})
+
+	t.Run("BelowMin", func(t *testing.T) {
+		if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Bitmap).Bits()) {
+			t.Fatalf("unexpected result: %s", spew.Sdump(result))
+		}
+	})
+
+	t.Run("AboveMax", func(t *testing.T) {
+		if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Bitmap).Bits()) {
+			t.Fatalf("unexpected result: %s", spew.Sdump(result))
+		}
+	})
+
+	t.Run("ErrFrameNotFound", func(t *testing.T) {
+		if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=bad_frame, foo >= 20)`), nil, nil); err != pilosa.ErrFrameNotFound {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("ErrFieldNotFound", func(t *testing.T) {
+		if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound {
+			t.Fatal(err)
+		}
+	})
+}
+
 // Ensure a remote query can return a bitmap.
 func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
 	c := test.NewCluster(2)

fragment.go

@@ -39,6 +39,7 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/internal"
+	"github.com/pilosa/pilosa/pql"
 	"github.com/pilosa/pilosa/roaring"
 )
 
@@ -567,14 +568,14 @@ func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, e
 	return sum, count, nil
 }
 
-func (f *Fragment) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) {
+func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) {
 	switch op {
-	case RangeOpEQ:
+	case pql.EQ:
 		return f.fieldRangeEQ(bitDepth, predicate)
-	case RangeOpLT, RangeOpLTE:
-		return f.fieldRangeLT(bitDepth, predicate, op == RangeOpLTE)
-	case RangeOpGT, RangeOpGTE:
-		return f.fieldRangeGT(bitDepth, predicate, op == RangeOpGTE)
+	case pql.LT, pql.LTE:
+		return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE)
+	case pql.GT, pql.GTE:
+		return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE)
 	default:
 		return nil, ErrInvalidRangeOperation
 	}

fragment_test.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/pql"
 	"github.com/pilosa/pilosa/test"
 )
 
@@ -275,7 +276,7 @@ func TestFragment_FieldRange(t *testing.T) {
 		}
 
 		// Query for equality.
-		if b, err := f.FieldRange(pilosa.RangeOpEQ, bitDepth, 300); err != nil {
+		if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) {
 			t.Fatalf("unexpected bits: %+v", b.Bits())
@@ -302,28 +303,28 @@ func TestFragment_FieldRange(t *testing.T) {
 		}
 
 		// Query for fields less than (ending with set bit).
-		if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 301); err != nil {
+		if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
 			t.Fatalf("unexpected bits: %+v", b.Bits())
 		}
 
 		// Query for fields less than (ending with unset bit).
-		if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 300); err != nil {
+		if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) {
 			t.Fatalf("unexpected bits: %+v", b.Bits())
 		}
 
 		// Query for fields less than or equal to (ending with set bit).
-		if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 301); err != nil {
+		if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) {
 			t.Fatalf("unexpected bits: %+v", b.Bits())
 		}
 
 		// Query for fields less than or equal to (ending with unset bit).
-		if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 300); err != nil {
+		if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
 			t.Fatalf("unexpected bits: %+v", b.Bits())
@@ -350,28 +351,28 @@ func TestFragment_FieldRange(t *testing.T) {
 		}
 
 		// Query for fields greater than (ending with unset bit).
-		if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 300); err != nil {
+		if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
 			t.Fatalf("unexpected bits: %+v", b.Bits())
 		}
 
 		// Query for fields greater than (ending with set bit).
-		if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 301); err != nil {
+		if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) {
 			t.Fatalf("unexpected bits: %+v", b.Bits())
 		}
 
 		// Query for fields greater than or equal to (ending with unset bit).
-		if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 300); err != nil {
+		if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) {
 			t.Fatalf("unexpected bits: %+v", b.Bits())
 		}
 
 		// Query for fields greater than or equal to (ending with set bit).
-		if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 301); err != nil {
+		if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
 			t.Fatalf("unexpected bits: %+v", b.Bits())

frame.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/internal"
+	"github.com/pilosa/pilosa/pql"
 )
 
 // Default frame settings.
@@ -40,15 +41,6 @@ const (
 	DefaultCacheSize = 50000
 )
 
-// List of operators for field range queries.
-const (
-	RangeOpEQ  = "eq"
-	RangeOpLT  = "lt"
-	RangeOpLTE = "lte"
-	RangeOpGT  = "gt"
-	RangeOpGTE = "gte"
-)
-
 // Frame represents a container for views.
 type Frame struct {
 	mu          sync.Mutex
@@ -663,7 +655,7 @@ func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err err
 	return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil
 }
 
-func (f *Frame) FieldRange(name, op string, predicate int64) (*Bitmap, error) {
+func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap, error) {
 	// Retrieve and validate field.
 	field := f.Field(name)
 	if field == nil {

pql/ast.go

@@ -198,6 +198,16 @@ func (c *Call) IsInverse(rowLabel, columnLabel string) bool {
 	return false
 }
 
+// HasConditionArg returns true if any arg is a conditional.
+func (c *Call) HasConditionArg() bool {
+	for _, v := range c.Args {
+		if _, ok := v.(*Condition); ok {
+			return true
+		}
+	}
+	return false
+}
+
 // Condition represents an operation & value.
 // When used in an argument map it represents a binary expression.
 type Condition struct {

view.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 
 	"github.com/pilosa/pilosa/internal"
+	"github.com/pilosa/pilosa/pql"
 )
 
 // View layout modes.
@@ -314,7 +315,7 @@ func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err e
 }
 
 // FieldRange returns bitmaps with a field value encoding matching the predicate.
-func (v *View) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) {
+func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) {
 	bm := NewBitmap()
 	for _, frag := range v.Fragments() {
 		other, err := frag.FieldRange(op, bitDepth, predicate)