Merge pull request #1849 from molecula/fb-1148

distinct on timestamps can reduce now

executor.go

@@ -1181,6 +1181,8 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string,
 			return other.Union(v.(*Row))
 		case nil:
 			return v
+		case DistinctTimestamp:
+			return other.Union(v.(DistinctTimestamp))
 		default:
 			return errors.Errorf("unexpected return type from executeDistinctShard: %+v %T", other, other)
 		}
@@ -1633,6 +1635,22 @@ type DistinctTimestamp struct {
 	Name   string
 }
 
+// Union returns the union of the values of `d` and `other`
+func (d *DistinctTimestamp) Union(other DistinctTimestamp) DistinctTimestamp {
+	both := map[string]string{}
+	for _, val := range d.Values {
+		both[val] = val
+	}
+	for _, val := range other.Values {
+		both[val] = val
+	}
+	vals := []string{}
+	for key := range both {
+		vals = append(vals, key)
+	}
+	return DistinctTimestamp{Name: d.Name, Values: vals}
+}
+
 func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result *Row, err0 error) {
 	index := idx.Name()
 	tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard})

executor_internal_test.go

@@ -504,3 +504,44 @@ func TestGetScaledInt(t *testing.T) {
 	}
 
 }
+
+func TestDistinctTimestampUnion(t *testing.T) {
+	cases := []struct {
+		name     string
+		a        DistinctTimestamp
+		b        DistinctTimestamp
+		expected DistinctTimestamp
+	}{
+		{
+			name:     "empty other",
+			a:        DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
+			b:        DistinctTimestamp{Name: "a", Values: []string{}},
+			expected: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
+		},
+		{
+			name:     "one more in other",
+			a:        DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c"}},
+			b:        DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c", "d"}},
+			expected: DistinctTimestamp{Name: "a", Values: []string{"a", "b", "c", "d"}},
+		},
+	}
+	for _, test := range cases {
+		t.Run(test.name, func(t *testing.T) {
+			res := test.a.Union(test.b)
+			allThere := true
+			for _, val := range res.Values {
+				here := false
+				for _, expected := range test.expected.Values {
+					if val == expected {
+						here = true
+						break
+					}
+				}
+				allThere = allThere && here
+			}
+			if !allThere {
+				t.Errorf("expected %v, got %v", test.expected, res)
+			}
+		})
+	}
+}

executor_test.go

@@ -6752,9 +6752,10 @@ func variousQueriesCountDistinctTimestamp(t *testing.T, c *test.Cluster) {
 	c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s"))
 
 	// add some data
-	data := []string{"2010-01-02T12:32:00Z", "2010-04-20T12:32:00Z", "2011-04-20T12:32:00Z"}
+	data := []string{"2010-01-02T12:32:00Z", "2010-04-20T12:32:00Z", "2011-04-20T12:59:00Z", "2011-04-20T12:40:00Z", "2011-04-20T12:32:00Z"}
+
 	for i, datum := range data {
-		c.Query(t, index, fmt.Sprintf("Set(%d, ts=\"%s\")", i+10, datum))
+		c.Query(t, index, fmt.Sprintf("Set(%d, ts=\"%s\")", i*(1<<20), datum))
 	}
 
 	// query the Count of Distinct vals in field ts