Merge branch 'master' into 54mir/authentication

executor.go

@@ -602,6 +602,11 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q
 			return nil, err
 		}
 
+		if vc, ok := v.(ValCount); ok {
+			vc.cleanup()
+			v = vc
+		}
+
 		results = append(results, v)
 		// Some Calls can have significant data associated with them
 		// that gets generated during processing, such as Precomputed
@@ -612,6 +617,22 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q
 	return results, nil
 }
 
+// cleanup removes the integer value (Val) from the ValCount if one of
+// the other fields is in use.
+//
+// ValCounts are normally holding data which is stored as a BSI
+// (integer) under the hood. Sometimes it's convenient to be able to
+// compare the underlying integer values rather than their
+// interpretation as decimal, timestamp, etc, so the lower level
+// functions may return both integer and the interpreted value, but we
+// don't want to pass that all the way back to the client, so we
+// remove it here.
+func (vc *ValCount) cleanup() {
+	if vc.Val != 0 && (vc.FloatVal != 0 || !vc.TimestampVal.IsZero() || vc.DecimalVal != nil) {
+		vc.Val = 0
+	}
+}
+
 // preprocessQuery expands any calls that need preprocessing.
 func (e *executor) preprocessQuery(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*pql.Call, error) {
 	switch c.Name {
@@ -1276,6 +1297,10 @@ func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string
 	if err != nil {
 		return ValCount{}, errors.New("Percentile(): field required")
 	}
+	field := e.Holder.Field(index, fieldName)
+	if field == nil {
+		return ValCount{}, ErrFieldNotFound
+	}
 
 	// filter call for min & max
 	var filterCall *pql.Call
@@ -1296,7 +1321,7 @@ func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string
 		return ValCount{}, errors.Wrap(err, "executing Min call for Percentile")
 	}
 	if nthFloat == 0.0 {
-		return ValCount{Val: minVal.Val, Count: minVal.Count}, nil
+		return minVal, nil
 	}
 
 	// get max
@@ -1363,11 +1388,11 @@ func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string
 		} else if leftCountWeighted < rightCount {
 			min = possibleNthVal + 1
 		} else {
-			return ValCount{Val: possibleNthVal, Count: 1}, nil
+			return field.valCountize(possibleNthVal, 1, nil)
 		}
 	}
 
-	return ValCount{Val: min, Count: 1}, nil
+	return field.valCountize(min, 1, nil)
 
 }
 
@@ -8146,6 +8171,8 @@ func getScaledInt(f *Field, v interface{}) (int64, error) {
 		switch tv := v.(type) {
 		case time.Time:
 			value = tv.UnixNano() / TimeUnitNanos(f.options.TimeUnit)
+		case int64:
+			value = tv
 		default:
 			return 0, errors.Errorf("unexpected timestamp value type %T, val %v", tv, tv)
 		}

executor_internal_test.go

@@ -489,3 +489,18 @@ func TestExecutorSafeCopyDistinctTimestamp(t *testing.T) {
 		t.Fatalf("Did not copy results. got %+v, want %+v", copied.Results, response.Results)
 	}
 }
+
+func TestGetScaledInt(t *testing.T) {
+	f := OpenField(t, OptFieldTypeTimestamp(time.Now(), "ms"))
+	defer f.Close()
+	// check that fields with type timestamp return the int64 passed in to getScaledInt with nil err
+	v := time.Now().Unix()
+	res, err := getScaledInt(f.Field, v)
+	if err != nil {
+		t.Errorf("got error %v, expected nil", err)
+	}
+	if !reflect.DeepEqual(res, v) {
+		t.Errorf("expected %v, got %v", v, res)
+	}
+
+}

field.go

@@ -1388,18 +1388,7 @@ func (f *Field) MaxForShard(tx Tx, shard uint64, filter *Row) (ValCount, error)
 		return ValCount{}, errors.Wrap(err, "calling fragment.max")
 	}
 
-	valCount := ValCount{Count: int64(cnt)}
-
-	if f.Options().Type == FieldTypeDecimal {
-		dec := pql.NewDecimal(max+bsig.Base, bsig.Scale)
-		valCount.DecimalVal = &dec
-	} else if f.Options().Type == FieldTypeTimestamp {
-		valCount.TimestampVal = time.Unix(0, (max+bsig.Base)*TimeUnitNanos(f.options.TimeUnit)).UTC()
-	} else {
-		valCount.Val = max + bsig.Base
-	}
-
-	return valCount, nil
+	return f.valCountize(max, cnt, bsig)
 }
 
 // MinForShard returns the minimum value which appears in this shard
@@ -1434,17 +1423,32 @@ func (f *Field) MinForShard(tx Tx, shard uint64, filter *Row) (ValCount, error)
 		return ValCount{}, errors.Wrap(err, "calling fragment.min")
 	}
 
+	return f.valCountize(min, cnt, bsig)
+}
+
+// valCountize takes the "raw" value and count we get from the
+// fragment and calculates the cooked values for this field
+// (timestamping, decimaling, or just adding in the base). It always
+// includes the int64 "Val\" value to make comparisons easier in the
+// executor (at time of writing, Percentile takes advantage of this,
+// but we might be able to simplify logic in other places as well).
+func (f *Field) valCountize(val int64, cnt uint64, bsig *bsiGroup) (ValCount, error) {
+	if bsig == nil {
+		bsig = f.bsiGroup(f.name)
+		if bsig == nil {
+			return ValCount{}, ErrBSIGroupNotFound
+		}
+
+	}
 	valCount := ValCount{Count: int64(cnt)}
 
 	if f.Options().Type == FieldTypeDecimal {
-		dec := pql.NewDecimal(min+bsig.Base, bsig.Scale)
+		dec := pql.NewDecimal(val+bsig.Base, bsig.Scale)
 		valCount.DecimalVal = &dec
 	} else if f.Options().Type == FieldTypeTimestamp {
-		valCount.TimestampVal = time.Unix(0, (min+bsig.Base)*TimeUnitNanos(f.options.TimeUnit)).UTC()
-	} else {
-		valCount.Val = min + bsig.Base
+		valCount.TimestampVal = time.Unix(0, (val+bsig.Base)*TimeUnitNanos(f.options.TimeUnit)).UTC()
 	}
-
+	valCount.Val = val + bsig.Base
 	return valCount, nil
 }
 

field_internal_test.go

@@ -182,6 +182,23 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 	})
 }
 
+func TestField_ValCountize(t *testing.T) {
+	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
+	// check that you get an empty val count and err
+	// BSIGroupNotFound on nil bsig from
+	// f.bsiGroup(f.name)
+	f.bsiGroups = []*bsiGroup{}
+	v, err := f.valCountize(42, 42, nil)
+	if !reflect.DeepEqual(v, ValCount{}) {
+		t.Errorf("expected %v, got %v", ValCount{}, v)
+	}
+	if err != ErrBSIGroupNotFound {
+		t.Errorf("expected %v, got %v", ErrBSIGroupNotFound, err)
+	}
+
+}
+
 // Ensure field can open and retrieve a view.
 func TestField_DeleteView(t *testing.T) {
 	f := OpenField(t, OptFieldTypeDefault())
@@ -748,29 +765,29 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
 			name:      "single",
 			columnIDs: []uint64{1},
 			values:    []float64{10.1},
-			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
-			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+			expMax:    ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+			expMin:    ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
 		},
 		{
 			name:      "twovals",
 			columnIDs: []uint64{1, 2},
 			values:    []float64{10.1, 20.2},
-			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 1},
-			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+			expMax:    ValCount{Val: 20200, DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 1},
+			expMin:    ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
 		},
 		{
 			name:      "multiplecounts",
 			columnIDs: []uint64{1, 2, 3, 4, 5},
 			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2},
-			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
-			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+			expMax:    ValCount{Val: 20200, DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
 		},
 		{
 			name:      "middlevals",
 			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
 			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11},
-			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
-			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+			expMax:    ValCount{Val: 20200, DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
 		},
 	} {
 		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {