Merge branch 'master' into seebs/setvaluebenchmarks

.circleci/config.yml

@@ -50,7 +50,7 @@ jobs:
       - *fast-checkout
       - run: sudo apt-get install lsof
       - run:
-          command: make test TESTFLAGS="-race -timeout=30m"
+          command: make test TESTFLAGS="-race -v -timeout=30m"
           no_output_timeout: 30m
   test-golang-1.11-386:
     <<: *base-test

docs/query-language.md

@@ -341,13 +341,12 @@ Row(stargazer=1)
 **Spec:**
 
 ```
-Row(<FIELD>=<ROW>, <TIMESTAMP>, <TIMESTAMP>)
+Row(<FIELD>=<ROW>, from=<TIMESTAMP>, to=<TIMESTAMP>)
 ```
 
 **Description:**
 
-Similar to `Row`, but only returns bits which were set with timestamps
-between the given `start` (first) and `end` (second) timestamps.
+Similar to `Row`, but only returns bits which were set with timestamps between the given `from` (inclusive) and `to` (exclusive) timestamps. Both `from` and `to` parameters are optional. The default for `to` timestamp is current time + 1 day. If a later end timestamp is required, specify it explicitly.
 
 **Result Type:** object with attrs and bits
 
@@ -356,7 +355,7 @@ between the given `start` (first) and `end` (second) timestamps.
 
 Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range:
 ```request
-Row(stargazer=1, 2010-01-01T00:00, 2017-03-02T03:00)
+Row(stargazer=1, from='2010-01-01T00:00', to='2017-03-02T03:00')
 ```
 ```response
 {{"attrs":{},"columns":[10]}
@@ -836,7 +835,7 @@ GroupBy(<RowsCall>, [RowsCall...], limit=<UINT>, filter=<CALL>)
 
 GroupBy returns the count of the intersection of every combination of rows
 taking one row each from the specified `Rows` calls. It returns only those
-combinations for which the count is greater than 0. 
+combinations for which the count is greater than 0.
 
 The optional `filter` argument takes any type of `Row` query (e.g. Row, Union,
  Intersect, etc.) which will be intersected with each result prior to returning

executor.go

@@ -1253,9 +1253,8 @@ func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Cal
 	// Set maximum "to" value if only "from" is set. We don't need to worry
 	// about setting the minimum "from" since it is the zero value if omitted.
 	if toTime.IsZero() {
-		// This is the maximum comparable time.Time value.
-		// https://stackoverflow.com/a/32620397
-		toTime = time.Unix(1<<63-62135596801, 999999999)
+		// Set the end timestamp to current time + 1 day, in order to account for timezone differences.
+		toTime = time.Now().AddDate(0, 0, 1)
 	}
 
 	// Union bitmaps across all time-based views.

executor_test.go

@@ -1480,19 +1480,25 @@ func TestExecutor_Execute_Sum(t *testing.T) {
 // Ensure a range query can be executed.
 func TestExecutor_Execute_Row_Range(t *testing.T) {
 	t.Run("RowIDColumnID", func(t *testing.T) {
-		writeQuery := `
+		// Create a timestamp just out of the current date + 1 day timestamp (default end timestamp).
+		nextDayExclusive := time.Now().AddDate(0, 0, 2)
+
+		writeQuery := fmt.Sprintf(`
 		Set(2, f=1, 1999-12-31T00:00)
 		Set(3, f=1, 2000-01-01T00:00)
 		Set(4, f=1, 2000-01-02T00:00)
 		Set(5, f=1, 2000-02-01T00:00)
 		Set(6, f=1, 2001-01-01T00:00)
 		Set(7, f=1, 2002-01-01T02:00)
+		Set(8, f=1, %s)
 
 		Set(2, f=1, 1999-12-30T00:00)
 		Set(2, f=1, 2002-02-01T00:00)
-		Set(2, f=10, 2001-01-01T00:00)`
+		Set(2, f=10, 2001-01-01T00:00)`, nextDayExclusive.Format("2006-01-02T15:04"))
 		readQueries := []string{
 			`Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
+			`Row(f=1, from=1999-12-31T00:00)`,
+			`Row(f=1, to=2002-01-01T02:00)`,
 			`Clear( 2, f=1)`,
 			`Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
 		}
@@ -1505,8 +1511,20 @@ func TestExecutor_Execute_Row_Range(t *testing.T) {
 			}
 		})
 
+		t.Run("From", func(t *testing.T) {
+			if columns := responses[1].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
+				t.Fatalf("unexpected columns: %+v", columns)
+			}
+		})
+
+		t.Run("To", func(t *testing.T) {
+			if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6}) {
+				t.Fatalf("unexpected columns: %+v", columns)
+			}
+		})
+
 		t.Run("Clear", func(t *testing.T) {
-			if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) {
+			if columns := responses[4].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) {
 				t.Fatalf("unexpected columns: %+v", columns)
 			}
 		})
@@ -1901,16 +1919,44 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) {
 	})
 
 	t.Run("BETWEEN", func(t *testing.T) {
-		if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(0 < other < 1000)`}); err != nil {
-			t.Fatal(err)
-		} else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) {
-			t.Fatalf("unexpected result: %s", spew.Sdump(result))
+		tests := []struct {
+			q   string
+			exp bool
+		}{
+			{q: `Row(0 < other < 1000)`, exp: false},
+			{q: `Row(0 <= other < 1000)`, exp: false},
+			{q: `Row(0 <= other <= 1000)`, exp: true},
+			{q: `Row(0 < other <= 1000)`, exp: true},
+
+			{q: `Row(1000 < other < 1000)`, exp: false},
+			{q: `Row(1000 <= other < 1000)`, exp: false},
+			{q: `Row(1000 <= other <= 1000)`, exp: true},
+			{q: `Row(1000 < other <= 1000)`, exp: false},
+
+			{q: `Row(1000 < other < 2000)`, exp: false},
+			{q: `Row(1000 <= other < 2000)`, exp: true},
+			{q: `Row(1000 <= other <= 2000)`, exp: true},
+			{q: `Row(1000 < other <= 2000)`, exp: false},
 		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("#%d_%s", i, test.q), func(t *testing.T) {
+				var expected = []uint64{}
+				if test.exp {
+					expected = []uint64{0}
+				}
+				if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil {
+					t.Fatal(err)
+				} else if !reflect.DeepEqual(expected, result.Results[0].(*pilosa.Row).Columns()) {
+					t.Fatalf("unexpected result for query: %s", test.q)
+				}
+			})
+		}
+
 	})
 
 	// Ensure that the NotNull code path gets run.
 	t.Run("NotNull", func(t *testing.T) {
-		if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(-1 < other < 1000)`}); err != nil {
+		if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(0 <= other <= 1000)`}); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) {
 			t.Fatalf("unexpected result: %s", spew.Sdump(result))
@@ -2069,14 +2115,14 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) {
 	t.Run("BETWEEN", func(t *testing.T) {
 		if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 < other < 1000)`}); err != nil {
 			t.Fatal(err)
-		} else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) {
+		} else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) {
 			t.Fatalf("unexpected result: %s", spew.Sdump(result))
 		}
 	})
 
 	// Ensure that the NotNull code path gets run.
 	t.Run("NotNull", func(t *testing.T) {
-		if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(-1 < other < 1000)`}); err != nil {
+		if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 <= other <= 1000)`}); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) {
 			t.Fatalf("unexpected result: %s", spew.Sdump(result))

fragment_internal_test.go

@@ -1221,7 +1221,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
 	if err := f.Open(); err != nil {
 		b.Fatal(err)
 	}
-	defer f.Clean(b)
+	defer f.CleanKeep(b)
 
 	// Reset timer and execute benchmark.
 	b.ResetTimer()
@@ -1750,7 +1750,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
 	if err := f.Open(); err != nil {
 		b.Fatal(err)
 	}
-	defer f.Clean(b)
+	defer f.CleanKeep(b)
 	b.ResetTimer()
 
 	// Reset timer and execute benchmark.
@@ -2109,6 +2109,20 @@ func (f *fragment) Clean(t testing.TB) {
 	}
 }
 
+// CleanKeep is just like Clean(), but it doesn't remove the
+// fragment file (note that it DOES remove the cache file).
+func (f *fragment) CleanKeep(t testing.TB) {
+	errc := f.Close()
+	errp := os.Remove(f.cachePath())
+	if errc != nil {
+		t.Fatal("closing fragment: ", errc, errp)
+	}
+	// not all fragments have cache files
+	if errp != nil && !os.IsNotExist(errp) {
+		t.Fatalf("cleaning up fragment cache: %v", errp)
+	}
+}
+
 // mustOpenFragment returns a new instance of Fragment with a temporary path.
 func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment {
 	file, err := ioutil.TempFile(TempDir, "pilosa-fragment-")

pql/ast.go

@@ -91,8 +91,8 @@ func (q *Query) endConditional() {
 	if q.conditional[1] == "<" {
 		low++
 	}
-	if q.conditional[3] == "<=" {
-		high++
+	if q.conditional[3] == "<" {
+		high--
 	}
 
 	elem := q.lastCallStackElem()

pql/pqlpeg_test.go

@@ -248,6 +248,14 @@ func TestPEGWorking(t *testing.T) {
 			name:   "RangeTimeQuotes",
 			input:  `Row(a=4, from='2010-07-04T00:00', to="2010-08-04T00:00")`,
 			ncalls: 1},
+		{
+			name:   "RangeTimeFromQuotes",
+			input:  `Row(a=4, from='2010-07-04T00:00')`,
+			ncalls: 1},
+		{
+			name:   "RangeTimeToQuotes",
+			input:  `Row(a=4, to="2010-08-04T00:00")`,
+			ncalls: 1},
 		{
 			name:   "Dashed Frame",
 			input:  "Set(1, my-frame=9)",
@@ -501,7 +509,7 @@ func TestPQLDeepEquality(t *testing.T) {
 				Args: map[string]interface{}{
 					"a": &Condition{
 						Op:    BETWEEN,
-						Value: []interface{}{int64(4), int64(9)},
+						Value: []interface{}{int64(4), int64(8)},
 					},
 				},
 			}},
@@ -513,7 +521,7 @@ func TestPQLDeepEquality(t *testing.T) {
 				Args: map[string]interface{}{
 					"a": &Condition{
 						Op:    BETWEEN,
-						Value: []interface{}{int64(5), int64(9)},
+						Value: []interface{}{int64(5), int64(8)},
 					},
 				},
 			}},
@@ -525,7 +533,7 @@ func TestPQLDeepEquality(t *testing.T) {
 				Args: map[string]interface{}{
 					"a": &Condition{
 						Op:    BETWEEN,
-						Value: []interface{}{int64(4), int64(10)},
+						Value: []interface{}{int64(4), int64(9)},
 					},
 				},
 			}},
@@ -537,7 +545,7 @@ func TestPQLDeepEquality(t *testing.T) {
 				Args: map[string]interface{}{
 					"a": &Condition{
 						Op:    BETWEEN,
-						Value: []interface{}{int64(5), int64(10)},
+						Value: []interface{}{int64(5), int64(9)},
 					},
 				},
 			}},
@@ -629,6 +637,26 @@ func TestPQLDeepEquality(t *testing.T) {
 					{Name: "Rows"},
 				},
 			}},
+		{
+			name: "GroupByFilterRangeLTLT",
+			call: "GroupBy(Rows(), filter=Row(4 < a < 9))",
+			exp: &Call{
+				Name: "GroupBy",
+				Args: map[string]interface{}{
+					"filter": &Call{
+						Name: "Row",
+						Args: map[string]interface{}{
+							"a": &Condition{
+								Op:    BETWEEN,
+								Value: []interface{}{int64(5), int64(8)},
+							},
+						},
+					},
+				},
+				Children: []*Call{
+					{Name: "Rows"},
+				},
+			}},
 	}
 
 	for i, test := range tests {