Merge branch 'master' into transactionmetrics

executor.go

@@ -2463,21 +2463,15 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c
 		return nil, ErrFieldNotFound
 	}
 
-	// EQ null           (not implemented: flip frag.NotNull with max ColumnID)
+	// EQ null           _exists - frag.NotNull()
 	// NEQ null          frag.NotNull()
 	// BETWEEN a,b(in)   BETWEEN/frag.RowBetween()
 	// BETWEEN a,b(out)  BETWEEN/frag.NotNull()
 	// EQ <int>          frag.RangeOp
 	// NEQ <int>         frag.RangeOp
 
-	// Handle `!= null`.
+	// Handle `!= null` and `== null`.
 	if cond.Op == pql.NEQ && cond.Value == nil {
-		// Find bsiGroup.
-		bsig := f.bsiGroup(fieldName)
-		if bsig == nil {
-			return nil, ErrBSIGroupNotFound
-		}
-
 		// Retrieve fragment.
 		frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
 		if frag == nil {
@@ -2486,6 +2480,37 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c
 
 		return frag.notNull()
 
+	} else if cond.Op == pql.EQ && cond.Value == nil {
+		// Make sure the index supports existence tracking.
+		idx := e.Holder.Index(index)
+		if idx == nil {
+			return nil, ErrIndexNotFound
+		} else if idx.existenceField() == nil {
+			return nil, errors.Errorf("index does not support existence tracking: %s", index)
+		}
+
+		var existenceRow *Row
+		existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard)
+		if existenceFrag == nil {
+			existenceRow = NewRow()
+		} else {
+			existenceRow = existenceFrag.row(0)
+		}
+
+		var notNull *Row
+		var err error
+
+		// Retrieve notNull from fragment if it exists.
+		if frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard); frag != nil {
+			if notNull, err = frag.notNull(); err != nil {
+				return nil, errors.Wrap(err, "getting fragment not null")
+			}
+		} else {
+			notNull = NewRow()
+		}
+
+		return existenceRow.Difference(notNull), nil
+
 	} else if cond.Op == pql.BETWEEN || cond.Op == pql.BTWN_LT_LT ||
 		cond.Op == pql.BTWN_LTE_LT || cond.Op == pql.BTWN_LT_LTE {
 		predicates, err := getCondIntSlice(f, cond)

executor_test.go

@@ -2371,7 +2371,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) {
 	defer c.Close()
 	hldr := test.Holder{Holder: c[0].Server.Holder()}
 
-	idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
+	idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{TrackExistence: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2414,6 +2414,19 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) {
 	}
 
 	t.Run("EQ", func(t *testing.T) {
+		// EQ null
+		if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other == null)`}); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual([]uint64{1,
+			50,
+			ShardWidth,
+			ShardWidth + 1,
+			ShardWidth + 2,
+			(5 * ShardWidth) + 100,
+		}, result.Results[0].(*pilosa.Row).Columns()) {
+			t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns())
+		}
+		// EQ <int>
 		if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 20)`}); err != nil {
 			t.Fatal(err)
 		} else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{50, (5 * ShardWidth) + 100}; !reflect.DeepEqual(exp, got) {

pql/parser_test.go

@@ -173,7 +173,7 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with condition arguments.
 	t.Run("WithCondition", func(t *testing.T) {
-		q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
+		q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null, n == null)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
@@ -185,6 +185,7 @@ func TestParser_Parse(t *testing.T) {
 					"y":   &pql.Condition{Op: pql.GTE, Value: int64(100)},
 					"z":   &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}},
 					"m":   &pql.Condition{Op: pql.NEQ, Value: nil},
+					"n":   &pql.Condition{Op: pql.EQ, Value: nil},
 				},
 			},
 		) {

pql/pqlpeg_test.go

@@ -238,8 +238,16 @@ func TestPEGWorking(t *testing.T) {
 			name:   "RangeEQ",
 			input:  "Row(a == 4)",
 			ncalls: 1},
+		{
+			name:   "RangeEQNULL",
+			input:  "Row(a == null)",
+			ncalls: 1},
 		{
 			name:   "RangeNEQ",
+			input:  "Row(a != 4)",
+			ncalls: 1},
+		{
+			name:   "RangeNEQNull",
 			input:  "Row(a != null)",
 			ncalls: 1},
 		{