diff --git a/executor.go b/executor.go index ae960a5e1..a620a1145 100644 --- a/executor.go +++ b/executor.go @@ -715,7 +715,30 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * fieldName, cond = k, vv } - if cond.Op == pql.BETWEEN { + // EQ null (not implemented: flip frag.FieldNotNull with max ColumnID) + // NEQ null frag.FieldNotNull() + // BETWEEN a,b(in) BETWEEN/frag.FieldRangeBetween() + // BETWEEN a,b(out) BETWEEN/frag.FieldNotNull() + // EQ frag.FieldRange + // NEQ frag.FieldRange + + // Handle `!= null`. + if cond.Op == pql.NEQ && cond.Value == nil { + // Find field. + field := f.Field(fieldName) + if field == nil { + return nil, ErrFieldNotFound + } + + // Retrieve fragment. + frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + if frag == nil { + return NewBitmap(), nil + } + + return frag.FieldNotNull(field.BitDepth()) + + } else if cond.Op == pql.BETWEEN { predicates, err := cond.IntSliceValue() if err != nil { @@ -727,6 +750,10 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * return nil, errors.New("Range(): BETWEEN condition requires exactly two integer values") } + // The reason we don't just call: + // return f.FieldRangeBetween(fieldName, predicates[0], predicates[1]) + // here is because we need the call to be slice-specific. + // Find field. field := f.Field(fieldName) if field == nil { @@ -744,6 +771,12 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * return NewBitmap(), nil } + // If the query is asking for the entire valid range, just return + // the not-null bitmap for the field. + if predicates[0] <= field.Min && predicates[1] >= field.Max { + return frag.FieldNotNull(field.BitDepth()) + } + return frag.FieldRangeBetween(field.BitDepth(), baseValueMin, baseValueMax) } else { @@ -761,7 +794,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * } baseValue, outOfRange := field.BaseValue(cond.Op, value) - if outOfRange { + if outOfRange && cond.Op != pql.NEQ { return NewBitmap(), nil } @@ -771,6 +804,11 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * return NewBitmap(), nil } + // outOfRange for NEQ should return all not-null. + if outOfRange && cond.Op == pql.NEQ { + return frag.FieldNotNull(field.BitDepth()) + } + f.Stats.Count("range:field", 1, 1.0) return frag.FieldRange(cond.Op, field.BitDepth(), baseValue) } diff --git a/executor_test.go b/executor_test.go index 482903bc2..3ccac27ec 100644 --- a/executor_test.go +++ b/executor_test.go @@ -764,6 +764,28 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } }) + t.Run("NEQ", func(t *testing.T) { + // NEQ null + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != null)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + // NEQ + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo != 20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + // NEQ - + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != -20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Bitmap).Bits()) { + //t.Fatalf("unexpected result: %s", spew.Sdump(result)) + t.Fatalf("unexpected result: %s", result[0].(*pilosa.Bitmap).Bits()) + } + }) + t.Run("LT", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil { t.Fatal(err) @@ -796,6 +818,23 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } }) + t.Run("BETWEEN", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [1, 1000])`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + // Ensure that the FieldNotNull code path gets run. + t.Run("FieldNotNull", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [0, 1000])`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + t.Run("BelowMin", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil { t.Fatal(err) diff --git a/fragment.go b/fragment.go index cca8c4ede..263966ad0 100644 --- a/fragment.go +++ b/fragment.go @@ -619,6 +619,8 @@ func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*B switch op { case pql.EQ: return f.fieldRangeEQ(bitDepth, predicate) + case pql.NEQ: + return f.fieldRangeNEQ(bitDepth, predicate) case pql.LT, pql.LTE: return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE) case pql.GT, pql.GTE: @@ -647,6 +649,22 @@ func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Bitmap, error return b, nil } +func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Bitmap, error) { + // Start with set of columns with values set. + b := f.Row(uint64(bitDepth)) + + // Get the equal bitmap. + eq, err := f.fieldRangeEQ(bitDepth, predicate) + if err != nil { + return nil, err + } + + // Not-null minus the equal bitmap. + b = b.Difference(eq) + + return b, nil +} + func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Bitmap, error) { keep := NewBitmap() @@ -730,6 +748,11 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b return b, nil } +// FieldNotNull returns the not-null row (stored at bitDepth). +func (f *Fragment) FieldNotNull(bitDepth uint) (*Bitmap, error) { + return f.Row(uint64(bitDepth)), nil +} + func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Bitmap, error) { b := f.Row(uint64(bitDepth)) keep1 := NewBitmap() // GTE diff --git a/fragment_test.go b/fragment_test.go index 21723dc3b..2ccf538f7 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -283,6 +283,29 @@ func TestFragment_FieldRange(t *testing.T) { } }) + t.Run("NEQ", func(t *testing.T) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + t.Fatal(err) + } + + // Query for inequality. + if b, err := f.FieldRange(pql.NEQ, bitDepth, 300); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + }) + t.Run("LT", func(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() diff --git a/frame.go b/frame.go index e7bfd4a40..141772a93 100644 --- a/frame.go +++ b/frame.go @@ -1119,7 +1119,7 @@ func (f *Field) BaseValue(op pql.Token, value int64) (baseValue uint64, outOfRan } else { baseValue = uint64(value - f.Min) } - } else if op == pql.EQ { + } else if op == pql.EQ || op == pql.NEQ { if value < f.Min || value > f.Max { return baseValue, true } diff --git a/pql/parser.go b/pql/parser.go index ab5298458..3af0cbc9c 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -165,7 +165,7 @@ func (p *Parser) parseArgs() (map[string]interface{}, error) { var op Token switch tok, pos, lit := p.scanIgnoreWhitespace(); tok { case ASSIGN: - case EQ, LT, LTE, GT, GTE, BETWEEN: + case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN: op = tok default: return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit) diff --git a/pql/parser_test.go b/pql/parser_test.go index 266e1de1f..0e2a5c17d 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -172,7 +172,7 @@ func TestParser_Parse(t *testing.T) { // Parse with condition arguments. t.Run("WithCondition", func(t *testing.T) { - q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8])`) + q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], @@ -183,6 +183,7 @@ func TestParser_Parse(t *testing.T) { "x": &pql.Condition{Op: pql.EQ, Value: 12.25}, "y": &pql.Condition{Op: pql.GTE, Value: int64(100)}, "z": &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}}, + "m": &pql.Condition{Op: pql.NEQ, Value: nil}, }, }, ) { diff --git a/pql/scanner.go b/pql/scanner.go index f25abf1f4..5a24b6af2 100644 --- a/pql/scanner.go +++ b/pql/scanner.go @@ -66,6 +66,12 @@ func (s *Scanner) Scan() (tok Token, pos Pos, lit string) { } s.unread() return ASSIGN, pos, string(ch) + case '!': + if next := s.read(); next == '=' { + return NEQ, pos, "!=" + } + s.unread() + return ASSIGN, pos, string(ch) case '<': if next := s.read(); next == '=' { return LTE, pos, "<=" diff --git a/pql/scanner_test.go b/pql/scanner_test.go index 98c13e232..e48896748 100644 --- a/pql/scanner_test.go +++ b/pql/scanner_test.go @@ -38,6 +38,7 @@ func TestScanner_Scan(t *testing.T) { {name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`}, {name: "EQ", s: `==`, tok: pql.EQ, lit: `==`}, + {name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`}, {name: "LT", s: `<`, tok: pql.LT, lit: `<`}, {name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`}, {name: "GT", s: `>`, tok: pql.GT, lit: `>`}, diff --git a/pql/token.go b/pql/token.go index 2870553e4..6997f17af 100644 --- a/pql/token.go +++ b/pql/token.go @@ -39,6 +39,7 @@ const ( ASSIGN // = EQ // == + NEQ // != LT // < LTE // <= GT // > @@ -64,6 +65,7 @@ var tokens = [...]string{ ASSIGN: "=", EQ: "==", + NEQ: "!=", LT: "<", LTE: "<=", GT: ">",