Merge pull request #1191 from travisturner/min-max-bsi

Implement Min/Max BSI queries
This commit is contained in:
Travis Turner 2018-04-24 11:00:35 -05:00 committed by GitHub
commit 47607d5d2e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 737 additions and 123 deletions

View file

@ -333,14 +333,44 @@ func TestClient_ImportValue(t *testing.T) {
t.Fatal(err)
}
// Verify Sum.
sum, cnt, err := frame.FieldSum(nil, fld.Name)
if err != nil {
t.Fatal(err)
}
// Verify data.
if sum != 50 || cnt != 3 {
t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=70, cnt=3", sum, cnt)
t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt)
}
// Verify Min.
min, cnt, err := frame.FieldMin(nil, fld.Name)
if err != nil {
t.Fatal(err)
}
if min != -10 || cnt != 1 {
t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt)
}
// Verify Min with Filter.
filter, err := frame.FieldRange(fld.Name, pql.GT, 40)
if err != nil {
t.Fatal(err)
}
min, cnt, err = frame.FieldMin(filter, fld.Name)
if err != nil {
t.Fatal(err)
}
if min != -100 || cnt != 0 {
t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt)
}
// Verify Max.
max, cnt, err := frame.FieldMax(nil, fld.Name)
if err != nil {
t.Fatal(err)
}
if max != 40 || cnt != 1 {
t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt)
}
}

View file

@ -98,7 +98,7 @@ SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00")
#### BSI Range-Encoding
Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded
bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers.
bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers.
Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.

View file

@ -14,7 +14,7 @@ nav = []
<strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap.
<strong id="bsi">[BSI](../data-model/#bsi-range-encoding)</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi) and [Sum](#sum) queries.
<strong id="bsi">[BSI](../data-model/#bsi-range-encoding)</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries.
<strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
@ -30,8 +30,12 @@ nav = []
<strong id="jump-consistent-hash">[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf):</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
<strong id="max">[Max](../query-language/#max):</strong> A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field).
<strong id="maxslice">MaxSlice:</strong> The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries.
<strong id="min">[Min](../query-language/#min):</strong> A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field).
<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a [cluster](#cluster).
<strong id="partition">Partition:</strong> The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data.

View file

@ -512,6 +512,56 @@ Range(frame="stats", commitactivity >< [100, 200])
This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way.
#### Min
**Spec:**
```
Min([BITMAP_CALL], <frame=STRING>, <field=STRING>)
```
**Description:**
Returns the minimum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all collumns are considered.
**Result Type:** object with the min and count of columns containing the min value.
**Examples:**
Query the size of all repositories.
```
Min(frame="stats", field="diskusage")
```
Return `{"min":4,"count":2}`
* Result is the smallest repository in kilobytes, plus the number of repositories of that size.
#### Max
**Spec:**
```
Max([BITMAP_CALL], <frame=STRING>, <field=STRING>)
```
**Description:**
Returns the maximum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
**Result Type:** object with the max and count of columns containing the max value.
**Examples:**
Query the size of all repositories.
```
Max(frame="stats", field="diskusage")
```
Return `{"max":88,"count":13}`
* Result is the largest repository in kilobytes, plus the number of repositories of that size.
#### Sum
**Spec:**

View file

@ -243,7 +243,7 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ
#### Introduction
Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range` and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
First, create an index called `patients`:
``` request
@ -328,7 +328,7 @@ curl localhost:10101/index/patients/query \
```
The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`.
You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40.
You can also provide a filter to the `Sum()` function to find the average age of all patients over 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
@ -339,6 +339,48 @@ curl localhost:10101/index/patients/query \
```
Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query.
To find the minimum age of all patients, run a `Min` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Min(frame="measurements", field="age")'
```
``` response
{"results":[{"min":19,"count":1}]}
```
The results you get from the `Min` query contain the `min` of all values as well as the `count` of columns with that value.
You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Min(Range(frame="measurements", age > 40), frame="measurements", field="age")'
```
``` response
{"results":[{"min":57,"count":1}]}
```
To find the maximum age of all patients, run a `Max` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Max(frame="measurements", field="age")'
```
``` response
{"results":[{"max":71,"count":1}]}
```
The results you get from the `Max` query contain the `max` of all values as well as the `count` of columns with that value.
You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Max(Range(frame="measurements", age < 40), frame="measurements", field="age")'
```
``` response
{"results":[{"max":34,"count":1}]}
```
### Storing Row and Column Attributes
#### Introduction

View file

@ -157,6 +157,12 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
case "Sum":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeSum(ctx, index, c, slices, opt)
case "Min":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeFieldMin(ctx, index, c, slices, opt)
case "Max":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeFieldMax(ctx, index, c, slices, opt)
case "ClearBit":
return e.executeClearBit(ctx, index, c, opt)
case "Count":
@ -199,15 +205,15 @@ func (e *Executor) validateCallArgs(c *pql.Call) error {
}
// executeSum executes a Sum() call.
func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (SumCount, error) {
func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return SumCount{}, errors.New("Sum(): frame required")
return ValCount{}, errors.New("Sum(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return SumCount{}, errors.New("Sum(): field required")
return ValCount{}, errors.New("Sum(): field required")
}
if len(c.Children) > 1 {
return SumCount{}, errors.New("Sum() only accepts a single bitmap input")
return ValCount{}, errors.New("Sum() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
@ -217,18 +223,88 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(SumCount)
return other.Add(v.(SumCount))
other, _ := prev.(ValCount)
return other.Add(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return SumCount{}, err
return ValCount{}, err
}
other, _ := result.(SumCount)
other, _ := result.(ValCount)
if other.Count == 0 {
return SumCount{}, nil
return ValCount{}, nil
}
return other, nil
}
// executeFieldMin executes a Min() call.
func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return ValCount{}, errors.New("Min(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Min(): field required")
}
if len(c.Children) > 1 {
return ValCount{}, errors.New("Min() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeFieldMinSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.Smaller(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return ValCount{}, err
}
other, _ := result.(ValCount)
if other.Count == 0 {
return ValCount{}, nil
}
return other, nil
}
// executeFieldMax executes a Max() call.
func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return ValCount{}, errors.New("Max(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Max(): field required")
}
if len(c.Children) > 1 {
return ValCount{}, errors.New("Max() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeFieldMaxSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.Larger(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return ValCount{}, err
}
other, _ := result.(ValCount)
if other.Count == 0 {
return ValCount{}, nil
}
return other, nil
}
@ -318,13 +394,13 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *
}
}
// executeSumCountSlice executes calculates the sum & count for fields on a slice.
func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) {
// executeSumCountSlice calculates the sum and count for fields on a slice.
func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return SumCount{}, err
return ValCount{}, err
}
filter = bm
}
@ -334,29 +410,107 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return SumCount{}, nil
return ValCount{}, nil
}
field := frame.Field(fieldName)
if field == nil {
return SumCount{}, nil
return ValCount{}, nil
}
view := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if view == nil {
return SumCount{}, nil
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
vsum, vcount, err := view.FieldSum(filter, field.BitDepth())
vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth())
if err != nil {
return SumCount{}, err
return ValCount{}, err
}
return SumCount{
Sum: int64(vsum) + (int64(vcount) * field.Min),
return ValCount{
Val: int64(vsum) + (int64(vcount) * field.Min),
Count: int64(vcount),
}, nil
}
// executeFieldMinSlice calculates the min for fields on a slice.
func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return ValCount{}, err
}
filter = bm
}
frameName, _ := c.Args["frame"].(string)
fieldName, _ := c.Args["field"].(string)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ValCount{}, nil
}
field := frame.Field(fieldName)
if field == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth())
if err != nil {
return ValCount{}, err
}
return ValCount{
Val: int64(fmin) + field.Min,
Count: int64(fcount),
}, nil
}
// executeFieldMaxSlice calculates the max for fields on a slice.
func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return ValCount{}, err
}
filter = bm
}
frameName, _ := c.Args["frame"].(string)
fieldName, _ := c.Args["field"].(string)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ValCount{}, nil
}
field := frame.Field(fieldName)
if field == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth())
if err != nil {
return ValCount{}, err
}
return ValCount{
Val: int64(fmax) + field.Min,
Count: int64(fcount),
}, nil
}
// executeTopN executes a TopN() call.
// This first performs the TopN() to determine the top results and then
// requeries to retrieve the full counts for each of the top results.
@ -1355,7 +1509,7 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *
switch call.Name {
case "Average", "Sum":
v, err = decodeSumCount(pb.Results[i].GetSumCount()), nil
v, err = decodeValCount(pb.Results[i].GetValCount()), nil
case "TopN":
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case "Count":
@ -1593,29 +1747,51 @@ func needsSlices(calls []*pql.Call) bool {
return false
}
// SumCount represents a grouping of sum & count for Sum() and Average() calls.
type SumCount struct {
Sum int64 `json:"sum"`
// ValCount represents a grouping of sum & count for Sum() and Average() calls.
type ValCount struct {
Val int64 `json:"value"`
Count int64 `json:"count"`
}
func (sc *SumCount) Add(other SumCount) SumCount {
return SumCount{
Sum: sc.Sum + other.Sum,
Count: sc.Count + other.Count,
func (vc *ValCount) Add(other ValCount) ValCount {
return ValCount{
Val: vc.Val + other.Val,
Count: vc.Count + other.Count,
}
}
func encodeSumCount(sc SumCount) *internal.SumCount {
return &internal.SumCount{
Sum: sc.Sum,
Count: sc.Count,
func encodeValCount(vc ValCount) *internal.ValCount {
return &internal.ValCount{
Val: vc.Val,
Count: vc.Count,
}
}
func decodeSumCount(pb *internal.SumCount) SumCount {
return SumCount{
Sum: pb.Sum,
func decodeValCount(pb *internal.ValCount) ValCount {
return ValCount{
Val: pb.Val,
Count: pb.Count,
}
}
// Smaller returns the smaller of the two ValCounts.
func (vc *ValCount) Smaller(other ValCount) ValCount {
if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) {
return other
}
return ValCount{
Val: vc.Val,
Count: vc.Count,
}
}
// Larger returns the larger of the two ValCounts.
func (vc *ValCount) Larger(other ValCount) ValCount {
if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) {
return other
}
return ValCount{
Val: vc.Val,
Count: vc.Count,
}
}

View file

@ -597,6 +597,98 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
}
}
// Ensure Min() and Max() queries can be executed.
func TestExecutor_Execute_MinMax(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100},
},
}); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=3)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(frame=f, row=1, col=1)
SetBit(frame=f, row=2, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, col=0)
SetFieldValue(frame=f, foo=-5, col=1)
SetFieldValue(frame=f, foo=-5, col=2)
SetFieldValue(frame=f, foo=10, col=3)
SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("Min", func(t *testing.T) {
tests := []struct {
filter string
exp int64
cnt int64
}{
{filter: ``, exp: -5, cnt: 2},
{filter: `Bitmap(frame=f, row=0)`, exp: 10, cnt: 1},
{filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
if tt.filter == "" {
pql = `Min(frame=f, field=foo)`
} else {
pql = fmt.Sprintf(`Min(%s, frame=f, field=foo)`, tt.filter)
}
if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) {
t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result))
}
}
})
t.Run("Max", func(t *testing.T) {
tests := []struct {
filter string
exp int64
cnt int64
}{
{filter: ``, exp: 60, cnt: 1},
{filter: `Bitmap(frame=f, row=0)`, exp: 60, cnt: 1},
{filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
if tt.filter == "" {
pql = `Max(frame=f, field=foo)`
} else {
pql = fmt.Sprintf(`Max(%s, frame=f, field=foo)`, tt.filter)
}
if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) {
t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result))
}
}
})
}
// Ensure a Sum() query can be executed.
func TestExecutor_Execute_Sum(t *testing.T) {
hldr := test.MustOpenHolder()
@ -642,7 +734,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
t.Run("NoFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 200, Count: 5}) {
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -650,7 +742,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
t.Run("WithFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 80, Count: 2}) {
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})

View file

@ -614,6 +614,70 @@ func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, e
return sum, count, nil
}
// FieldMin returns the min of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
// If there are no columns to consider, return early.
if consider.Count() == 0 {
return 0, 0, nil
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
row := f.Row(uint64(ii))
x := consider.Difference(row)
count = x.Count()
if count > 0 {
consider = x
} else {
min += (1 << ii)
if ii == 0 {
count = consider.Count()
}
}
}
return min, count, nil
}
// FieldMax returns the max of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
// If there are no columns to consider, return early.
if consider.Count() == 0 {
return 0, 0, nil
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
row := f.Row(uint64(ii))
x := row.Intersect(consider)
count = x.Count()
if count > 0 {
max += (1 << ii)
consider = x
} else if ii == 0 {
count = consider.Count()
}
}
return max, count, nil
}
// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) {
switch op {

View file

@ -256,6 +256,79 @@ func TestFragment_FieldSum(t *testing.T) {
})
}
// Ensure a fragment can find the min and max of field values.
func TestFragment_FieldMinMax(t *testing.T) {
const bitDepth = 16
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil {
t.Fatal(err)
}
t.Run("Min", func(t *testing.T) {
tests := []struct {
filter *pilosa.Bitmap
exp uint64
cnt uint64
}{
{filter: nil, exp: 0, cnt: 1},
{filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 300, cnt: 2},
{filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2},
{filter: pilosa.NewBitmap(1), exp: 0, cnt: 0},
{filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1},
{filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1},
}
for i, test := range tests {
if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil {
t.Fatal(err)
} else if min != test.exp {
t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min)
} else if cnt != test.cnt {
t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt)
}
}
})
t.Run("Max", func(t *testing.T) {
tests := []struct {
filter *pilosa.Bitmap
exp uint64
cnt uint64
}{
{filter: nil, exp: 2818, cnt: 2},
{filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 2818, cnt: 1},
{filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2},
{filter: pilosa.NewBitmap(1), exp: 0, cnt: 0},
{filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1},
{filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1},
}
for i, test := range tests {
if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil {
t.Fatal(err)
} else if max != test.exp {
t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max)
} else if cnt != test.cnt {
t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt)
}
}
})
}
// Ensure a fragment query for matching fields.
func TestFragment_FieldRange(t *testing.T) {
const bitDepth = 16

View file

@ -737,6 +737,46 @@ func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err err
return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil
}
// FieldMin returns the min for a field.
// An optional filtering bitmap can be provided.
func (f *Frame) FieldMin(filter *Bitmap, name string) (min, count int64, err error) {
field := f.Field(name)
if field == nil {
return 0, 0, ErrFieldNotFound
}
view := f.View(ViewFieldPrefix + name)
if view == nil {
return 0, 0, nil
}
vmin, vcount, err := view.FieldMin(filter, field.BitDepth())
if err != nil {
return 0, 0, err
}
return int64(vmin) + field.Min, int64(vcount), nil
}
// FieldMax returns the max for a field.
// An optional filtering bitmap can be provided.
func (f *Frame) FieldMax(filter *Bitmap, name string) (max, count int64, err error) {
field := f.Field(name)
if field == nil {
return 0, 0, ErrFieldNotFound
}
view := f.View(ViewFieldPrefix + name)
if view == nil {
return 0, 0, nil
}
vmax, vcount, err := view.FieldMax(filter, field.BitDepth())
if err != nil {
return 0, 0, err
}
return int64(vmax) + field.Min, int64(vcount), nil
}
func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap, error) {
// Retrieve and validate field.
field := f.Field(name)

View file

@ -1205,7 +1205,7 @@ const (
QueryResultTypeNil uint32 = iota
QueryResultTypeBitmap
QueryResultTypePairs
QueryResultTypeSumCount
QueryResultTypeValCount
QueryResultTypeUint64
QueryResultTypeBool
)
@ -1294,9 +1294,9 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse {
case []Pair:
pb.Results[i].Type = QueryResultTypePairs
pb.Results[i].Pairs = encodePairs(result)
case SumCount:
pb.Results[i].Type = QueryResultTypeSumCount
pb.Results[i].SumCount = encodeSumCount(result)
case ValCount:
pb.Results[i].Type = QueryResultTypeValCount
pb.Results[i].ValCount = encodeValCount(result)
case uint64:
pb.Results[i].Type = QueryResultTypeUint64
pb.Results[i].N = result

View file

@ -11,7 +11,7 @@
It has these top-level messages:
Bitmap
Pair
SumCount
ValCount
Bit
ColumnAttrSet
Attr
@ -105,24 +105,24 @@ func (m *Pair) GetCount() uint64 {
return 0
}
type SumCount struct {
Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"`
type ValCount struct {
Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"`
Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"`
}
func (m *SumCount) Reset() { *m = SumCount{} }
func (m *SumCount) String() string { return proto.CompactTextString(m) }
func (*SumCount) ProtoMessage() {}
func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
func (m *ValCount) Reset() { *m = ValCount{} }
func (m *ValCount) String() string { return proto.CompactTextString(m) }
func (*ValCount) ProtoMessage() {}
func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
func (m *SumCount) GetSum() int64 {
func (m *ValCount) GetVal() int64 {
if m != nil {
return m.Sum
return m.Val
}
return 0
}
func (m *SumCount) GetCount() int64 {
func (m *ValCount) GetCount() int64 {
if m != nil {
return m.Count
}
@ -358,7 +358,7 @@ type QueryResult struct {
Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"`
N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"`
SumCount *SumCount `protobuf:"bytes,5,opt,name=SumCount" json:"SumCount,omitempty"`
ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"`
Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"`
}
@ -395,9 +395,9 @@ func (m *QueryResult) GetPairs() []*Pair {
return nil
}
func (m *QueryResult) GetSumCount() *SumCount {
func (m *QueryResult) GetValCount() *ValCount {
if m != nil {
return m.SumCount
return m.ValCount
}
return nil
}
@ -548,7 +548,7 @@ func (m *ImportValueRequest) GetValues() []int64 {
func init() {
proto.RegisterType((*Bitmap)(nil), "internal.Bitmap")
proto.RegisterType((*Pair)(nil), "internal.Pair")
proto.RegisterType((*SumCount)(nil), "internal.SumCount")
proto.RegisterType((*ValCount)(nil), "internal.ValCount")
proto.RegisterType((*Bit)(nil), "internal.Bit")
proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet")
proto.RegisterType((*Attr)(nil), "internal.Attr")
@ -655,7 +655,7 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *SumCount) Marshal() (dAtA []byte, err error) {
func (m *ValCount) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
@ -665,15 +665,15 @@ func (m *SumCount) Marshal() (dAtA []byte, err error) {
return dAtA[:n], nil
}
func (m *SumCount) MarshalTo(dAtA []byte) (int, error) {
func (m *ValCount) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if m.Sum != 0 {
if m.Val != 0 {
dAtA[i] = 0x8
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Sum))
i = encodeVarintPublic(dAtA, i, uint64(m.Val))
}
if m.Count != 0 {
dAtA[i] = 0x10
@ -1023,11 +1023,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.SumCount != nil {
if m.ValCount != nil {
dAtA[i] = 0x2a
i++
i = encodeVarintPublic(dAtA, i, uint64(m.SumCount.Size()))
n6, err := m.SumCount.MarshalTo(dAtA[i:])
i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size()))
n6, err := m.ValCount.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
@ -1317,11 +1317,11 @@ func (m *Pair) Size() (n int) {
return n
}
func (m *SumCount) Size() (n int) {
func (m *ValCount) Size() (n int) {
var l int
_ = l
if m.Sum != 0 {
n += 1 + sovPublic(uint64(m.Sum))
if m.Val != 0 {
n += 1 + sovPublic(uint64(m.Val))
}
if m.Count != 0 {
n += 1 + sovPublic(uint64(m.Count))
@ -1471,8 +1471,8 @@ func (m *QueryResult) Size() (n int) {
if m.Changed {
n += 2
}
if m.SumCount != nil {
l = m.SumCount.Size()
if m.ValCount != nil {
l = m.ValCount.Size()
n += 1 + l + sovPublic(uint64(l))
}
if m.Type != 0 {
@ -1874,7 +1874,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *SumCount) Unmarshal(dAtA []byte) error {
func (m *ValCount) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
@ -1897,17 +1897,17 @@ func (m *SumCount) Unmarshal(dAtA []byte) error {
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: SumCount: wiretype end group for non-group")
return fmt.Errorf("proto: ValCount: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: SumCount: illegal tag %d (wire type %d)", fieldNum, wire)
return fmt.Errorf("proto: ValCount: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Val", wireType)
}
m.Sum = 0
m.Val = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
@ -1917,7 +1917,7 @@ func (m *SumCount) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.Sum |= (int64(b) & 0x7F) << shift
m.Val |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -2959,7 +2959,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error {
m.Changed = bool(v != 0)
case 5:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field SumCount", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ValCount", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
@ -2983,10 +2983,10 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.SumCount == nil {
m.SumCount = &SumCount{}
if m.ValCount == nil {
m.ValCount = &ValCount{}
}
if err := m.SumCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
if err := m.ValCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
@ -3821,47 +3821,47 @@ var fileDescriptorPublic = []byte{
// 705 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40,
0x14, 0x65, 0x62, 0x27, 0x71, 0x6e, 0x92, 0xaa, 0x1a, 0x41, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85,
0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x29, 0x65, 0xed, 0xb6, 0xa3,
0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x2d, 0x61, 0xed, 0xb6, 0xa3,
0x62, 0xc9, 0x2f, 0xec, 0xb1, 0xda, 0x7c, 0x07, 0x1b, 0x3e, 0x81, 0x8f, 0x60, 0xc5, 0x0a, 0x76,
0x7c, 0x02, 0x94, 0x1f, 0x41, 0xf7, 0x8e, 0x27, 0x76, 0x5a, 0x09, 0x58, 0xb0, 0x9b, 0x73, 0xce,
0xcc, 0xf5, 0x9c, 0xb9, 0xe7, 0x26, 0x30, 0xca, 0xab, 0xb3, 0x38, 0x3a, 0x9f, 0xe6, 0x45, 0xa6,
0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x29, 0xf4, 0xe6, 0x91, 0x4a, 0xc2, 0x9c,
0x73, 0xb0, 0xe7, 0x91, 0x2a, 0x5d, 0xe6, 0x59, 0x81, 0x2d, 0x68, 0xcd, 0x9f, 0x40, 0xf7, 0x85,
0x52, 0x45, 0xe9, 0x76, 0x3c, 0x2b, 0x18, 0xce, 0xb6, 0xa6, 0xe6, 0xdc, 0x14, 0x69, 0xa1, 0x45,
0x3c, 0x79, 0x24, 0x57, 0xa5, 0x6b, 0x79, 0x56, 0x30, 0x10, 0xb4, 0xf6, 0x9f, 0x81, 0xfd, 0x2a,
0x8c, 0x0a, 0xbe, 0x05, 0x9d, 0xc5, 0xbe, 0xcb, 0x3c, 0x16, 0xd8, 0xa2, 0xb3, 0xd8, 0xe7, 0xf7,
0xa1, 0xbb, 0x97, 0x55, 0xa9, 0x72, 0x3b, 0x44, 0x69, 0xc0, 0xb7, 0xc1, 0x3a, 0x92, 0x2b, 0xd7,
0xf2, 0x58, 0x30, 0x10, 0xb8, 0xf4, 0x67, 0xe0, 0x2c, 0xab, 0x64, 0xad, 0x2e, 0xab, 0x84, 0x8a,
0x58, 0x02, 0x97, 0x9b, 0x55, 0xac, 0xba, 0x8a, 0xff, 0x06, 0xac, 0x79, 0xa4, 0x50, 0x14, 0xd9,
0xd5, 0xfa, 0xab, 0x1a, 0xf0, 0x47, 0xe0, 0xec, 0x65, 0x71, 0x95, 0xa4, 0x8b, 0xfd, 0xfa, 0xdb,
0x6b, 0xcc, 0x1f, 0xc3, 0xe0, 0x24, 0x4a, 0x64, 0xa9, 0xc2, 0x24, 0xa7, 0x4b, 0x58, 0xa2, 0x21,
0xfc, 0xb7, 0x30, 0xd6, 0x3b, 0xd1, 0xed, 0x52, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a,
0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xac, 0x72, 0x59, 0xdf, 0x94,
0xd6, 0xdc, 0x83, 0xe1, 0x52, 0x15, 0x51, 0x7a, 0x79, 0x1a, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29,
0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x12, 0x7a, 0xf3, 0x48, 0x25, 0x61, 0xce,
0x39, 0xd8, 0xf3, 0x48, 0x95, 0x2e, 0xf3, 0xac, 0xc0, 0x16, 0xb4, 0xe6, 0x4f, 0xa0, 0xfb, 0x42,
0xa9, 0xa2, 0x74, 0x3b, 0x9e, 0x15, 0x0c, 0x67, 0x5b, 0x53, 0x73, 0x6e, 0x8a, 0xb4, 0xd0, 0x22,
0x9e, 0x3c, 0x92, 0xab, 0xd2, 0xb5, 0x3c, 0x2b, 0x18, 0x08, 0x5a, 0xfb, 0xcf, 0xc0, 0x7e, 0x15,
0x46, 0x05, 0xdf, 0x82, 0xce, 0x62, 0xdf, 0x65, 0x1e, 0x0b, 0x6c, 0xd1, 0x59, 0xec, 0xf3, 0xfb,
0xd0, 0xdd, 0xcb, 0xaa, 0x54, 0xb9, 0x1d, 0xa2, 0x34, 0xe0, 0xdb, 0x60, 0x1d, 0xc9, 0x95, 0x6b,
0x79, 0x2c, 0x18, 0x08, 0x5c, 0xfa, 0x33, 0x70, 0x96, 0x61, 0xbc, 0x56, 0x97, 0x61, 0x4c, 0x45,
0x2c, 0x81, 0xcb, 0xcd, 0x2a, 0x56, 0x5d, 0xc5, 0x7f, 0x03, 0xd6, 0x3c, 0x52, 0x28, 0x8a, 0xec,
0x6a, 0xfd, 0x55, 0x0d, 0xf8, 0x23, 0x70, 0xf6, 0xb2, 0xb8, 0x4a, 0xd2, 0xc5, 0x7e, 0xfd, 0xed,
0x35, 0xe6, 0x8f, 0x61, 0x70, 0x1a, 0x25, 0xb2, 0x54, 0x61, 0x92, 0xd3, 0x25, 0x2c, 0xd1, 0x10,
0xfe, 0x5b, 0x18, 0xeb, 0x9d, 0xe8, 0xf6, 0x44, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a,
0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xae, 0x72, 0x59, 0xdf, 0x94,
0xd6, 0xdc, 0x83, 0xe1, 0x89, 0x2a, 0xa2, 0xf4, 0x72, 0x19, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29,
0xf4, 0xb8, 0x48, 0x95, 0x96, 0x6d, 0xb2, 0xb1, 0xc6, 0xe8, 0x71, 0x9e, 0x65, 0xb1, 0x16, 0xbb,
0x1e, 0x0b, 0x1c, 0xd1, 0x10, 0x7c, 0x02, 0x70, 0x18, 0x67, 0x61, 0x7d, 0xb6, 0xe7, 0xb1, 0x80,
0x89, 0x16, 0xe3, 0xef, 0x42, 0x1f, 0x6f, 0xfa, 0x32, 0xcc, 0x1b, 0xb7, 0xec, 0x0f, 0x6e, 0xfd,
0xcf, 0x0c, 0x46, 0xaf, 0x2b, 0x59, 0xac, 0x84, 0x7c, 0x5f, 0xc9, 0x92, 0xba, 0x42, 0xb8, 0x76,
0xa9, 0x01, 0xdf, 0x81, 0xde, 0x32, 0x8e, 0xce, 0xa5, 0x7e, 0x3b, 0x5b, 0xd4, 0x08, 0xbd, 0x36,
0x6f, 0x5e, 0x92, 0x57, 0x47, 0xb4, 0x29, 0x3c, 0x29, 0x64, 0x92, 0x29, 0x63, 0xa6, 0x46, 0xdc,
0x87, 0xd1, 0xc1, 0xf5, 0x79, 0x5c, 0x5d, 0x48, 0x7d, 0xb4, 0x47, 0xea, 0x06, 0x87, 0xd5, 0x6b,
0x4c, 0x89, 0xef, 0xeb, 0xea, 0x2d, 0xca, 0xff, 0xc0, 0x60, 0x5c, 0x5f, 0xbf, 0xcc, 0xb3, 0xb4,
0x94, 0xd8, 0xa3, 0x83, 0xa2, 0x30, 0x3d, 0x3a, 0x28, 0x0a, 0xbe, 0x0b, 0x7d, 0x21, 0xcb, 0x2a,
0x56, 0xa6, 0xf1, 0x0f, 0x9a, 0xa7, 0x30, 0x67, 0xab, 0x58, 0x09, 0xb3, 0x8b, 0x3f, 0x87, 0xad,
0x8d, 0x20, 0xe9, 0x89, 0x19, 0xce, 0x1e, 0x36, 0xe7, 0x36, 0x74, 0x71, 0x6b, 0xbb, 0xff, 0x8d,
0xc1, 0xb0, 0x55, 0x99, 0x07, 0x66, 0x78, 0xe9, 0x5a, 0xc3, 0xd9, 0x76, 0x53, 0x48, 0xf3, 0xc2,
0x0c, 0xf7, 0x08, 0xd8, 0x71, 0x1d, 0x26, 0x76, 0x8c, 0x2d, 0xc4, 0xe1, 0x34, 0xdf, 0x6f, 0xb5,
0x10, 0x69, 0xa1, 0x45, 0xee, 0x42, 0x7f, 0xef, 0x5d, 0x98, 0x5e, 0xca, 0x0b, 0x0a, 0x93, 0x23,
0x0c, 0xe4, 0xd3, 0x66, 0x38, 0xe9, 0xf5, 0x87, 0x33, 0xde, 0x94, 0x30, 0x8a, 0x68, 0x06, 0xd8,
0xa4, 0x19, 0x7b, 0x31, 0xd6, 0x69, 0xf6, 0x7f, 0x32, 0x18, 0x2f, 0x92, 0x3c, 0x2b, 0x54, 0x2b,
0x21, 0x8b, 0xf4, 0x42, 0x5e, 0x9b, 0x84, 0x10, 0x40, 0xf6, 0xb0, 0x08, 0x13, 0x3d, 0x0a, 0x03,
0xa1, 0x01, 0xb2, 0x94, 0x14, 0x4a, 0x86, 0x2d, 0x34, 0xa0, 0x4c, 0xe0, 0xb0, 0x97, 0xae, 0xad,
0xd3, 0xa4, 0x11, 0x66, 0xdf, 0xcc, 0x7a, 0xe9, 0x76, 0x49, 0x6a, 0x08, 0xcc, 0xfe, 0x7a, 0xd8,
0x31, 0x2f, 0x56, 0x60, 0x89, 0x16, 0x83, 0xef, 0x20, 0xb2, 0x2b, 0xfa, 0x85, 0xeb, 0xd3, 0x2f,
0x9c, 0x81, 0x78, 0x52, 0x97, 0x21, 0xd1, 0x21, 0xb1, 0xc5, 0xf8, 0x5f, 0x18, 0x70, 0xed, 0x91,
0xa6, 0xe8, 0xff, 0x19, 0xc5, 0xbd, 0x91, 0x8c, 0x75, 0x63, 0x70, 0x2f, 0x82, 0xbf, 0xd8, 0xdc,
0x81, 0x1e, 0xdd, 0xc2, 0x58, 0xac, 0xd1, 0x2d, 0x13, 0xfd, 0xdb, 0x26, 0xe6, 0xdb, 0x5f, 0x6f,
0x26, 0xec, 0xfb, 0xcd, 0x84, 0xfd, 0xb8, 0x99, 0xb0, 0x8f, 0xbf, 0x26, 0xf7, 0xce, 0x7a, 0xf4,
0x27, 0xf2, 0xf4, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa3, 0xa0, 0xd2, 0x51, 0x54, 0x06, 0x00,
0xa9, 0x01, 0xdf, 0x81, 0xde, 0x49, 0x1c, 0x9d, 0x4b, 0xfd, 0x76, 0xb6, 0xa8, 0x11, 0x7a, 0x6d,
0xde, 0xbc, 0x24, 0xaf, 0x8e, 0x68, 0x53, 0x78, 0x52, 0xc8, 0x24, 0x53, 0xc6, 0x4c, 0x8d, 0xb8,
0x0f, 0xa3, 0x83, 0xeb, 0xf3, 0xb8, 0xba, 0x90, 0xfa, 0x68, 0x8f, 0xd4, 0x0d, 0x0e, 0xab, 0xd7,
0x98, 0x12, 0xdf, 0xd7, 0xd5, 0x5b, 0x94, 0xff, 0x81, 0xc1, 0xb8, 0xbe, 0x7e, 0x99, 0x67, 0x69,
0x29, 0xb1, 0x47, 0x07, 0x45, 0x61, 0x7a, 0x74, 0x50, 0x14, 0x7c, 0x17, 0xfa, 0x42, 0x96, 0x55,
0xac, 0x4c, 0xe3, 0x1f, 0x34, 0x4f, 0x61, 0xce, 0x56, 0xb1, 0x12, 0x66, 0x17, 0x7f, 0x0e, 0x5b,
0x1b, 0x41, 0xd2, 0x13, 0x33, 0x9c, 0x3d, 0x6c, 0xce, 0x6d, 0xe8, 0xe2, 0xd6, 0x76, 0xff, 0x1b,
0x83, 0x61, 0xab, 0x32, 0x0f, 0xcc, 0xf0, 0xd2, 0xb5, 0x86, 0xb3, 0xed, 0xa6, 0x90, 0xe6, 0x85,
0x19, 0xee, 0x11, 0xb0, 0xe3, 0x3a, 0x4c, 0xec, 0x18, 0x5b, 0x88, 0xc3, 0x69, 0xbe, 0xdf, 0x6a,
0x21, 0xd2, 0x42, 0x8b, 0xdc, 0x85, 0xfe, 0xde, 0xbb, 0x30, 0xbd, 0x94, 0x17, 0x14, 0x26, 0x47,
0x18, 0xc8, 0xa7, 0xcd, 0x70, 0xd2, 0xeb, 0x0f, 0x67, 0xbc, 0x29, 0x61, 0x14, 0xd1, 0x0c, 0xb0,
0x49, 0x33, 0xf6, 0x62, 0xac, 0xd3, 0xec, 0xff, 0x64, 0x30, 0x5e, 0x24, 0x79, 0x56, 0xa8, 0x56,
0x42, 0x16, 0xe9, 0x85, 0xbc, 0x36, 0x09, 0x21, 0x80, 0xec, 0x61, 0x11, 0x26, 0x7a, 0x14, 0x06,
0x42, 0x03, 0x64, 0x29, 0x29, 0x94, 0x0c, 0x5b, 0x68, 0x40, 0x99, 0xc0, 0x61, 0x2f, 0x5d, 0x5b,
0xa7, 0x49, 0x23, 0xcc, 0xbe, 0x99, 0xf5, 0xd2, 0xed, 0x92, 0xd4, 0x10, 0x98, 0xfd, 0xf5, 0xb0,
0x63, 0x5e, 0xac, 0xc0, 0x12, 0x2d, 0x06, 0xdf, 0x41, 0x64, 0x57, 0xf4, 0x0b, 0xd7, 0xa7, 0x5f,
0x38, 0x03, 0xf1, 0xa4, 0x2e, 0x43, 0xa2, 0x43, 0x62, 0x8b, 0xf1, 0xbf, 0x30, 0xe0, 0xda, 0x23,
0x4d, 0xd1, 0xff, 0x33, 0x8a, 0x7b, 0x23, 0x19, 0xeb, 0xc6, 0xe0, 0x5e, 0x04, 0x7f, 0xb1, 0xb9,
0x03, 0x3d, 0xba, 0x85, 0xb1, 0x58, 0xa3, 0x5b, 0x26, 0xfa, 0xb7, 0x4d, 0xcc, 0xb7, 0xbf, 0xde,
0x4c, 0xd8, 0xf7, 0x9b, 0x09, 0xfb, 0x71, 0x33, 0x61, 0x1f, 0x7f, 0x4d, 0xee, 0x9d, 0xf5, 0xe8,
0x4f, 0xe4, 0xe9, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x3e, 0xba, 0x23, 0x54, 0x06, 0x00,
0x00,
}

View file

@ -14,8 +14,8 @@ message Pair {
uint64 Count = 2;
}
message SumCount {
int64 Sum = 1;
message ValCount {
int64 Val = 1;
int64 Count = 2;
}
@ -64,7 +64,7 @@ message QueryResult {
Bitmap Bitmap = 1;
uint64 N = 2;
repeated Pair Pairs = 3;
SumCount SumCount = 5;
ValCount ValCount = 5;
bool Changed = 4;
}

43
view.go
View file

@ -357,6 +357,49 @@ func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err e
return sum, count, nil
}
// FieldMin returns the min and count of a field.
func (v *View) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) {
var minHasValue bool
for _, f := range v.Fragments() {
fmin, fcount, err := f.FieldMin(filter, bitDepth)
if err != nil {
return min, count, err
}
// Don't consider a min based on zero columns.
if fcount == 0 {
continue
}
if !minHasValue {
min = fmin
minHasValue = true
count += fcount
continue
}
if fmin < min {
min = fmin
count += fcount
}
}
return min, count, nil
}
// FieldMax returns the max and count of a field.
func (v *View) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) {
for _, f := range v.Fragments() {
fmax, fcount, err := f.FieldMax(filter, bitDepth)
if err != nil {
return max, count, err
}
if fcount > 0 && fmax > max {
max = fmax
count += fcount
}
}
return max, count, nil
}
// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) {
bm := NewBitmap()