diff --git a/client_test.go b/client_test.go index d11ab16a9..be3505acd 100644 --- a/client_test.go +++ b/client_test.go @@ -333,14 +333,44 @@ func TestClient_ImportValue(t *testing.T) { t.Fatal(err) } + // Verify Sum. sum, cnt, err := frame.FieldSum(nil, fld.Name) if err != nil { t.Fatal(err) } - - // Verify data. if sum != 50 || cnt != 3 { - t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=70, cnt=3", sum, cnt) + t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt) + } + + // Verify Min. + min, cnt, err := frame.FieldMin(nil, fld.Name) + if err != nil { + t.Fatal(err) + } + if min != -10 || cnt != 1 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt) + } + + // Verify Min with Filter. + filter, err := frame.FieldRange(fld.Name, pql.GT, 40) + if err != nil { + t.Fatal(err) + } + min, cnt, err = frame.FieldMin(filter, fld.Name) + if err != nil { + t.Fatal(err) + } + if min != -100 || cnt != 0 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt) + } + + // Verify Max. + max, cnt, err := frame.FieldMax(nil, fld.Name) + if err != nil { + t.Fatal(err) + } + if max != 40 || cnt != 1 { + t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt) } } diff --git a/docs/data-model.md b/docs/data-model.md index 98750eda8..8375ae15f 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -98,7 +98,7 @@ SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00") #### BSI Range-Encoding Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded -bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers. +bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. diff --git a/docs/glossary.md b/docs/glossary.md index bd9293c3d..9b883017d 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -14,7 +14,7 @@ nav = [] [Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap. -[BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi) and [Sum](#sum) queries. +[BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries. Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. @@ -30,8 +30,12 @@ nav = [] [Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf): A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. +[Max](../query-language/#max): A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field). + MaxSlice: The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. +[Min](../query-language/#min): A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field). + Node: An individual running instance of Pilosa server which belongs to a [cluster](#cluster). Partition: The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data. diff --git a/docs/query-language.md b/docs/query-language.md index 83a10ab36..8ed0d5439 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -512,6 +512,56 @@ Range(frame="stats", commitactivity >< [100, 200]) This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way. +#### Min + +**Spec:** + +``` +Min([BITMAP_CALL], , ) +``` + +**Description:** + +Returns the minimum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all collumns are considered. + +**Result Type:** object with the min and count of columns containing the min value. + +**Examples:** + +Query the size of all repositories. +``` +Min(frame="stats", field="diskusage") +``` + +Return `{"min":4,"count":2}` + +* Result is the smallest repository in kilobytes, plus the number of repositories of that size. + +#### Max + +**Spec:** + +``` +Max([BITMAP_CALL], , ) +``` + +**Description:** + +Returns the maximum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered. + +**Result Type:** object with the max and count of columns containing the max value. + +**Examples:** + +Query the size of all repositories. +``` +Max(frame="stats", field="diskusage") +``` + +Return `{"max":88,"count":13}` + +* Result is the largest repository in kilobytes, plus the number of repositories of that size. + #### Sum **Spec:** diff --git a/docs/tutorials.md b/docs/tutorials.md index 601825fa9..39e345dbd 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -243,7 +243,7 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ #### Introduction -Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range` and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. +Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. First, create an index called `patients`: ``` request @@ -328,7 +328,7 @@ curl localhost:10101/index/patients/query \ ``` The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. -You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40. +You can also provide a filter to the `Sum()` function to find the average age of all patients over 40. ``` request curl localhost:10101/index/patients/query \ -X POST \ @@ -339,6 +339,48 @@ curl localhost:10101/index/patients/query \ ``` Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. +To find the minimum age of all patients, run a `Min` query: +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Min(frame="measurements", field="age")' +``` +``` response +{"results":[{"min":19,"count":1}]} +``` +The results you get from the `Min` query contain the `min` of all values as well as the `count` of columns with that value. + +You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40. +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Min(Range(frame="measurements", age > 40), frame="measurements", field="age")' +``` +``` response +{"results":[{"min":57,"count":1}]} +``` + +To find the maximum age of all patients, run a `Max` query: +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Max(frame="measurements", field="age")' +``` +``` response +{"results":[{"max":71,"count":1}]} +``` +The results you get from the `Max` query contain the `max` of all values as well as the `count` of columns with that value. + +You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40. +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Max(Range(frame="measurements", age < 40), frame="measurements", field="age")' +``` +``` response +{"results":[{"max":34,"count":1}]} +``` + ### Storing Row and Column Attributes #### Introduction diff --git a/executor.go b/executor.go index b7108e9e0..62bc0e8c8 100644 --- a/executor.go +++ b/executor.go @@ -157,6 +157,12 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "Sum": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeSum(ctx, index, c, slices, opt) + case "Min": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeFieldMin(ctx, index, c, slices, opt) + case "Max": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeFieldMax(ctx, index, c, slices, opt) case "ClearBit": return e.executeClearBit(ctx, index, c, opt) case "Count": @@ -199,15 +205,15 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { } // executeSum executes a Sum() call. -func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (SumCount, error) { +func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame, _ := c.Args["frame"]; frame == "" { - return SumCount{}, errors.New("Sum(): frame required") + return ValCount{}, errors.New("Sum(): frame required") } else if field, _ := c.Args["field"]; field == "" { - return SumCount{}, errors.New("Sum(): field required") + return ValCount{}, errors.New("Sum(): field required") } if len(c.Children) > 1 { - return SumCount{}, errors.New("Sum() only accepts a single bitmap input") + return ValCount{}, errors.New("Sum() only accepts a single bitmap input") } // Execute calls in bulk on each remote node and merge. @@ -217,18 +223,88 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.(SumCount) - return other.Add(v.(SumCount)) + other, _ := prev.(ValCount) + return other.Add(v.(ValCount)) } result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) if err != nil { - return SumCount{}, err + return ValCount{}, err } - other, _ := result.(SumCount) + other, _ := result.(ValCount) if other.Count == 0 { - return SumCount{}, nil + return ValCount{}, nil + } + return other, nil +} + +// executeFieldMin executes a Min() call. +func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { + if frame, _ := c.Args["frame"]; frame == "" { + return ValCount{}, errors.New("Min(): frame required") + } else if field, _ := c.Args["field"]; field == "" { + return ValCount{}, errors.New("Min(): field required") + } + + if len(c.Children) > 1 { + return ValCount{}, errors.New("Min() only accepts a single bitmap input") + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(slice uint64) (interface{}, error) { + return e.executeFieldMinSlice(ctx, index, c, slice) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(ValCount) + return other.Smaller(v.(ValCount)) + } + + result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + if err != nil { + return ValCount{}, err + } + other, _ := result.(ValCount) + + if other.Count == 0 { + return ValCount{}, nil + } + return other, nil +} + +// executeFieldMax executes a Max() call. +func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { + if frame, _ := c.Args["frame"]; frame == "" { + return ValCount{}, errors.New("Max(): frame required") + } else if field, _ := c.Args["field"]; field == "" { + return ValCount{}, errors.New("Max(): field required") + } + + if len(c.Children) > 1 { + return ValCount{}, errors.New("Max() only accepts a single bitmap input") + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(slice uint64) (interface{}, error) { + return e.executeFieldMaxSlice(ctx, index, c, slice) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(ValCount) + return other.Larger(v.(ValCount)) + } + + result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + if err != nil { + return ValCount{}, err + } + other, _ := result.(ValCount) + + if other.Count == 0 { + return ValCount{}, nil } return other, nil } @@ -318,13 +394,13 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * } } -// executeSumCountSlice executes calculates the sum & count for fields on a slice. -func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) { +// executeSumCountSlice calculates the sum and count for fields on a slice. +func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Bitmap if len(c.Children) == 1 { bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) if err != nil { - return SumCount{}, err + return ValCount{}, err } filter = bm } @@ -334,29 +410,107 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq frame := e.Holder.Frame(index, frameName) if frame == nil { - return SumCount{}, nil + return ValCount{}, nil } field := frame.Field(fieldName) if field == nil { - return SumCount{}, nil + return ValCount{}, nil } - view := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) - if view == nil { - return SumCount{}, nil + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { + return ValCount{}, nil } - vsum, vcount, err := view.FieldSum(filter, field.BitDepth()) + vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth()) if err != nil { - return SumCount{}, err + return ValCount{}, err } - return SumCount{ - Sum: int64(vsum) + (int64(vcount) * field.Min), + return ValCount{ + Val: int64(vsum) + (int64(vcount) * field.Min), Count: int64(vcount), }, nil } +// executeFieldMinSlice calculates the min for fields on a slice. +func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { + var filter *Bitmap + if len(c.Children) == 1 { + bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + if err != nil { + return ValCount{}, err + } + filter = bm + } + + frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) + + frame := e.Holder.Frame(index, frameName) + if frame == nil { + return ValCount{}, nil + } + + field := frame.Field(fieldName) + if field == nil { + return ValCount{}, nil + } + + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { + return ValCount{}, nil + } + + fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth()) + if err != nil { + return ValCount{}, err + } + return ValCount{ + Val: int64(fmin) + field.Min, + Count: int64(fcount), + }, nil +} + +// executeFieldMaxSlice calculates the max for fields on a slice. +func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { + var filter *Bitmap + if len(c.Children) == 1 { + bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + if err != nil { + return ValCount{}, err + } + filter = bm + } + + frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) + + frame := e.Holder.Frame(index, frameName) + if frame == nil { + return ValCount{}, nil + } + + field := frame.Field(fieldName) + if field == nil { + return ValCount{}, nil + } + + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { + return ValCount{}, nil + } + + fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth()) + if err != nil { + return ValCount{}, err + } + return ValCount{ + Val: int64(fmax) + field.Min, + Count: int64(fcount), + }, nil +} + // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. @@ -1355,7 +1509,7 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * switch call.Name { case "Average", "Sum": - v, err = decodeSumCount(pb.Results[i].GetSumCount()), nil + v, err = decodeValCount(pb.Results[i].GetValCount()), nil case "TopN": v, err = decodePairs(pb.Results[i].GetPairs()), nil case "Count": @@ -1593,29 +1747,51 @@ func needsSlices(calls []*pql.Call) bool { return false } -// SumCount represents a grouping of sum & count for Sum() and Average() calls. -type SumCount struct { - Sum int64 `json:"sum"` +// ValCount represents a grouping of sum & count for Sum() and Average() calls. +type ValCount struct { + Val int64 `json:"value"` Count int64 `json:"count"` } -func (sc *SumCount) Add(other SumCount) SumCount { - return SumCount{ - Sum: sc.Sum + other.Sum, - Count: sc.Count + other.Count, +func (vc *ValCount) Add(other ValCount) ValCount { + return ValCount{ + Val: vc.Val + other.Val, + Count: vc.Count + other.Count, } } -func encodeSumCount(sc SumCount) *internal.SumCount { - return &internal.SumCount{ - Sum: sc.Sum, - Count: sc.Count, +func encodeValCount(vc ValCount) *internal.ValCount { + return &internal.ValCount{ + Val: vc.Val, + Count: vc.Count, } } -func decodeSumCount(pb *internal.SumCount) SumCount { - return SumCount{ - Sum: pb.Sum, +func decodeValCount(pb *internal.ValCount) ValCount { + return ValCount{ + Val: pb.Val, Count: pb.Count, } } + +// Smaller returns the smaller of the two ValCounts. +func (vc *ValCount) Smaller(other ValCount) ValCount { + if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) { + return other + } + return ValCount{ + Val: vc.Val, + Count: vc.Count, + } +} + +// Larger returns the larger of the two ValCounts. +func (vc *ValCount) Larger(other ValCount) ValCount { + if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) { + return other + } + return ValCount{ + Val: vc.Val, + Count: vc.Count, + } +} diff --git a/executor_test.go b/executor_test.go index 1f8a38b99..0fd8a925b 100644 --- a/executor_test.go +++ b/executor_test.go @@ -597,6 +597,98 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { } } +// Ensure Min() and Max() queries can be executed. +func TestExecutor_Execute_MinMax(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=3) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=1, col=1) + SetBit(frame=f, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) + + SetFieldValue(frame=f, foo=20, col=0) + SetFieldValue(frame=f, foo=-5, col=1) + SetFieldValue(frame=f, foo=-5, col=2) + SetFieldValue(frame=f, foo=10, col=3) + SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + `), nil, nil); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: -5, cnt: 2}, + {filter: `Bitmap(frame=f, row=0)`, exp: 10, cnt: 1}, + {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Min(frame=f, field=foo)` + } else { + pql = fmt.Sprintf(`Min(%s, frame=f, field=foo)`, tt.filter) + } + if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 60, cnt: 1}, + {filter: `Bitmap(frame=f, row=0)`, exp: 60, cnt: 1}, + {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Max(frame=f, field=foo)` + } else { + pql = fmt.Sprintf(`Max(%s, frame=f, field=foo)`, tt.filter) + } + if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) +} + // Ensure a Sum() query can be executed. func TestExecutor_Execute_Sum(t *testing.T) { hldr := test.MustOpenHolder() @@ -642,7 +734,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Run("NoFilter", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 200, Count: 5}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -650,7 +742,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Run("WithFilter", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 80, Count: 2}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) diff --git a/fragment.go b/fragment.go index 56342690c..448ba3202 100644 --- a/fragment.go +++ b/fragment.go @@ -614,6 +614,70 @@ func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, e return sum, count, nil } +// FieldMin returns the min of a given field as well as the number of columns involved. +// A bitmap can be passed in to optionally filter the computed columns. +func (f *Fragment) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) { + + consider := f.Row(uint64(bitDepth)) + if filter != nil { + consider = consider.Intersect(filter) + } + + // If there are no columns to consider, return early. + if consider.Count() == 0 { + return 0, 0, nil + } + + for i := bitDepth; i > uint(0); i-- { + ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + row := f.Row(uint64(ii)) + + x := consider.Difference(row) + count = x.Count() + if count > 0 { + consider = x + } else { + min += (1 << ii) + if ii == 0 { + count = consider.Count() + } + } + } + + return min, count, nil +} + +// FieldMax returns the max of a given field as well as the number of columns involved. +// A bitmap can be passed in to optionally filter the computed columns. +func (f *Fragment) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) { + + consider := f.Row(uint64(bitDepth)) + if filter != nil { + consider = consider.Intersect(filter) + } + + // If there are no columns to consider, return early. + if consider.Count() == 0 { + return 0, 0, nil + } + + for i := bitDepth; i > uint(0); i-- { + ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + row := f.Row(uint64(ii)) + + x := row.Intersect(consider) + count = x.Count() + if count > 0 { + max += (1 << ii) + consider = x + } else if ii == 0 { + count = consider.Count() + } + } + + return max, count, nil +} + // FieldRange returns bitmaps with a field value encoding matching the predicate. func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { switch op { diff --git a/fragment_test.go b/fragment_test.go index 6a4a2f377..d7c97a5cf 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -256,6 +256,79 @@ func TestFragment_FieldSum(t *testing.T) { }) } +// Ensure a fragment can find the min and max of field values. +func TestFragment_FieldMinMax(t *testing.T) { + const bitDepth = 16 + + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter *pilosa.Bitmap + exp uint64 + cnt uint64 + }{ + {filter: nil, exp: 0, cnt: 1}, + {filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(1), exp: 0, cnt: 0}, + {filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1}, + {filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1}, + } + for i, test := range tests { + if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil { + t.Fatal(err) + } else if min != test.exp { + t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) + } else if cnt != test.cnt { + t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter *pilosa.Bitmap + exp uint64 + cnt uint64 + }{ + {filter: nil, exp: 2818, cnt: 2}, + {filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 2818, cnt: 1}, + {filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(1), exp: 0, cnt: 0}, + {filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1}, + {filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1}, + } + for i, test := range tests { + if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil { + t.Fatal(err) + } else if max != test.exp { + t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) + } else if cnt != test.cnt { + t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt) + } + } + }) +} + // Ensure a fragment query for matching fields. func TestFragment_FieldRange(t *testing.T) { const bitDepth = 16 diff --git a/frame.go b/frame.go index 6ed76d61c..0c87210bf 100644 --- a/frame.go +++ b/frame.go @@ -737,6 +737,46 @@ func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err err return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil } +// FieldMin returns the min for a field. +// An optional filtering bitmap can be provided. +func (f *Frame) FieldMin(filter *Bitmap, name string) (min, count int64, err error) { + field := f.Field(name) + if field == nil { + return 0, 0, ErrFieldNotFound + } + + view := f.View(ViewFieldPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmin, vcount, err := view.FieldMin(filter, field.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmin) + field.Min, int64(vcount), nil +} + +// FieldMax returns the max for a field. +// An optional filtering bitmap can be provided. +func (f *Frame) FieldMax(filter *Bitmap, name string) (max, count int64, err error) { + field := f.Field(name) + if field == nil { + return 0, 0, ErrFieldNotFound + } + + view := f.View(ViewFieldPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmax, vcount, err := view.FieldMax(filter, field.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmax) + field.Min, int64(vcount), nil +} + func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap, error) { // Retrieve and validate field. field := f.Field(name) diff --git a/handler.go b/handler.go index 94504679f..7bf11af1b 100644 --- a/handler.go +++ b/handler.go @@ -1205,7 +1205,7 @@ const ( QueryResultTypeNil uint32 = iota QueryResultTypeBitmap QueryResultTypePairs - QueryResultTypeSumCount + QueryResultTypeValCount QueryResultTypeUint64 QueryResultTypeBool ) @@ -1294,9 +1294,9 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { case []Pair: pb.Results[i].Type = QueryResultTypePairs pb.Results[i].Pairs = encodePairs(result) - case SumCount: - pb.Results[i].Type = QueryResultTypeSumCount - pb.Results[i].SumCount = encodeSumCount(result) + case ValCount: + pb.Results[i].Type = QueryResultTypeValCount + pb.Results[i].ValCount = encodeValCount(result) case uint64: pb.Results[i].Type = QueryResultTypeUint64 pb.Results[i].N = result diff --git a/internal/public.pb.go b/internal/public.pb.go index dfb9a819e..31447b9f0 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -11,7 +11,7 @@ It has these top-level messages: Bitmap Pair - SumCount + ValCount Bit ColumnAttrSet Attr @@ -105,24 +105,24 @@ func (m *Pair) GetCount() uint64 { return 0 } -type SumCount struct { - Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"` +type ValCount struct { + Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"` Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *SumCount) Reset() { *m = SumCount{} } -func (m *SumCount) String() string { return proto.CompactTextString(m) } -func (*SumCount) ProtoMessage() {} -func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (m *ValCount) Reset() { *m = ValCount{} } +func (m *ValCount) String() string { return proto.CompactTextString(m) } +func (*ValCount) ProtoMessage() {} +func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } -func (m *SumCount) GetSum() int64 { +func (m *ValCount) GetVal() int64 { if m != nil { - return m.Sum + return m.Val } return 0 } -func (m *SumCount) GetCount() int64 { +func (m *ValCount) GetCount() int64 { if m != nil { return m.Count } @@ -358,7 +358,7 @@ type QueryResult struct { Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - SumCount *SumCount `protobuf:"bytes,5,opt,name=SumCount" json:"SumCount,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` } @@ -395,9 +395,9 @@ func (m *QueryResult) GetPairs() []*Pair { return nil } -func (m *QueryResult) GetSumCount() *SumCount { +func (m *QueryResult) GetValCount() *ValCount { if m != nil { - return m.SumCount + return m.ValCount } return nil } @@ -548,7 +548,7 @@ func (m *ImportValueRequest) GetValues() []int64 { func init() { proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") - proto.RegisterType((*SumCount)(nil), "internal.SumCount") + proto.RegisterType((*ValCount)(nil), "internal.ValCount") proto.RegisterType((*Bit)(nil), "internal.Bit") proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") proto.RegisterType((*Attr)(nil), "internal.Attr") @@ -655,7 +655,7 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *SumCount) Marshal() (dAtA []byte, err error) { +func (m *ValCount) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -665,15 +665,15 @@ func (m *SumCount) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *SumCount) MarshalTo(dAtA []byte) (int, error) { +func (m *ValCount) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if m.Sum != 0 { + if m.Val != 0 { dAtA[i] = 0x8 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Sum)) + i = encodeVarintPublic(dAtA, i, uint64(m.Val)) } if m.Count != 0 { dAtA[i] = 0x10 @@ -1023,11 +1023,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.SumCount != nil { + if m.ValCount != nil { dAtA[i] = 0x2a i++ - i = encodeVarintPublic(dAtA, i, uint64(m.SumCount.Size())) - n6, err := m.SumCount.MarshalTo(dAtA[i:]) + i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size())) + n6, err := m.ValCount.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -1317,11 +1317,11 @@ func (m *Pair) Size() (n int) { return n } -func (m *SumCount) Size() (n int) { +func (m *ValCount) Size() (n int) { var l int _ = l - if m.Sum != 0 { - n += 1 + sovPublic(uint64(m.Sum)) + if m.Val != 0 { + n += 1 + sovPublic(uint64(m.Val)) } if m.Count != 0 { n += 1 + sovPublic(uint64(m.Count)) @@ -1471,8 +1471,8 @@ func (m *QueryResult) Size() (n int) { if m.Changed { n += 2 } - if m.SumCount != nil { - l = m.SumCount.Size() + if m.ValCount != nil { + l = m.ValCount.Size() n += 1 + l + sovPublic(uint64(l)) } if m.Type != 0 { @@ -1874,7 +1874,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } return nil } -func (m *SumCount) Unmarshal(dAtA []byte) error { +func (m *ValCount) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1897,17 +1897,17 @@ func (m *SumCount) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: SumCount: wiretype end group for non-group") + return fmt.Errorf("proto: ValCount: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: SumCount: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ValCount: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Val", wireType) } - m.Sum = 0 + m.Val = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -1917,7 +1917,7 @@ func (m *SumCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Sum |= (int64(b) & 0x7F) << shift + m.Val |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -2959,7 +2959,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { m.Changed = bool(v != 0) case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SumCount", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ValCount", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -2983,10 +2983,10 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.SumCount == nil { - m.SumCount = &SumCount{} + if m.ValCount == nil { + m.ValCount = &ValCount{} } - if err := m.SumCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.ValCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -3821,47 +3821,47 @@ var fileDescriptorPublic = []byte{ // 705 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40, 0x14, 0x65, 0x62, 0x27, 0x71, 0x6e, 0x92, 0xaa, 0x1a, 0x41, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, - 0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x29, 0x65, 0xed, 0xb6, 0xa3, + 0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x2d, 0x61, 0xed, 0xb6, 0xa3, 0x62, 0xc9, 0x2f, 0xec, 0xb1, 0xda, 0x7c, 0x07, 0x1b, 0x3e, 0x81, 0x8f, 0x60, 0xc5, 0x0a, 0x76, 0x7c, 0x02, 0x94, 0x1f, 0x41, 0xf7, 0x8e, 0x27, 0x76, 0x5a, 0x09, 0x58, 0xb0, 0x9b, 0x73, 0xce, 0xcc, 0xf5, 0x9c, 0xb9, 0xe7, 0x26, 0x30, 0xca, 0xab, 0xb3, 0x38, 0x3a, 0x9f, 0xe6, 0x45, 0xa6, - 0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x29, 0xf4, 0xe6, 0x91, 0x4a, 0xc2, 0x9c, - 0x73, 0xb0, 0xe7, 0x91, 0x2a, 0x5d, 0xe6, 0x59, 0x81, 0x2d, 0x68, 0xcd, 0x9f, 0x40, 0xf7, 0x85, - 0x52, 0x45, 0xe9, 0x76, 0x3c, 0x2b, 0x18, 0xce, 0xb6, 0xa6, 0xe6, 0xdc, 0x14, 0x69, 0xa1, 0x45, - 0x3c, 0x79, 0x24, 0x57, 0xa5, 0x6b, 0x79, 0x56, 0x30, 0x10, 0xb4, 0xf6, 0x9f, 0x81, 0xfd, 0x2a, - 0x8c, 0x0a, 0xbe, 0x05, 0x9d, 0xc5, 0xbe, 0xcb, 0x3c, 0x16, 0xd8, 0xa2, 0xb3, 0xd8, 0xe7, 0xf7, - 0xa1, 0xbb, 0x97, 0x55, 0xa9, 0x72, 0x3b, 0x44, 0x69, 0xc0, 0xb7, 0xc1, 0x3a, 0x92, 0x2b, 0xd7, - 0xf2, 0x58, 0x30, 0x10, 0xb8, 0xf4, 0x67, 0xe0, 0x2c, 0xab, 0x64, 0xad, 0x2e, 0xab, 0x84, 0x8a, - 0x58, 0x02, 0x97, 0x9b, 0x55, 0xac, 0xba, 0x8a, 0xff, 0x06, 0xac, 0x79, 0xa4, 0x50, 0x14, 0xd9, - 0xd5, 0xfa, 0xab, 0x1a, 0xf0, 0x47, 0xe0, 0xec, 0x65, 0x71, 0x95, 0xa4, 0x8b, 0xfd, 0xfa, 0xdb, - 0x6b, 0xcc, 0x1f, 0xc3, 0xe0, 0x24, 0x4a, 0x64, 0xa9, 0xc2, 0x24, 0xa7, 0x4b, 0x58, 0xa2, 0x21, - 0xfc, 0xb7, 0x30, 0xd6, 0x3b, 0xd1, 0xed, 0x52, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a, - 0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xac, 0x72, 0x59, 0xdf, 0x94, - 0xd6, 0xdc, 0x83, 0xe1, 0x52, 0x15, 0x51, 0x7a, 0x79, 0x1a, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29, + 0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x12, 0x7a, 0xf3, 0x48, 0x25, 0x61, 0xce, + 0x39, 0xd8, 0xf3, 0x48, 0x95, 0x2e, 0xf3, 0xac, 0xc0, 0x16, 0xb4, 0xe6, 0x4f, 0xa0, 0xfb, 0x42, + 0xa9, 0xa2, 0x74, 0x3b, 0x9e, 0x15, 0x0c, 0x67, 0x5b, 0x53, 0x73, 0x6e, 0x8a, 0xb4, 0xd0, 0x22, + 0x9e, 0x3c, 0x92, 0xab, 0xd2, 0xb5, 0x3c, 0x2b, 0x18, 0x08, 0x5a, 0xfb, 0xcf, 0xc0, 0x7e, 0x15, + 0x46, 0x05, 0xdf, 0x82, 0xce, 0x62, 0xdf, 0x65, 0x1e, 0x0b, 0x6c, 0xd1, 0x59, 0xec, 0xf3, 0xfb, + 0xd0, 0xdd, 0xcb, 0xaa, 0x54, 0xb9, 0x1d, 0xa2, 0x34, 0xe0, 0xdb, 0x60, 0x1d, 0xc9, 0x95, 0x6b, + 0x79, 0x2c, 0x18, 0x08, 0x5c, 0xfa, 0x33, 0x70, 0x96, 0x61, 0xbc, 0x56, 0x97, 0x61, 0x4c, 0x45, + 0x2c, 0x81, 0xcb, 0xcd, 0x2a, 0x56, 0x5d, 0xc5, 0x7f, 0x03, 0xd6, 0x3c, 0x52, 0x28, 0x8a, 0xec, + 0x6a, 0xfd, 0x55, 0x0d, 0xf8, 0x23, 0x70, 0xf6, 0xb2, 0xb8, 0x4a, 0xd2, 0xc5, 0x7e, 0xfd, 0xed, + 0x35, 0xe6, 0x8f, 0x61, 0x70, 0x1a, 0x25, 0xb2, 0x54, 0x61, 0x92, 0xd3, 0x25, 0x2c, 0xd1, 0x10, + 0xfe, 0x5b, 0x18, 0xeb, 0x9d, 0xe8, 0xf6, 0x44, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a, + 0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xae, 0x72, 0x59, 0xdf, 0x94, + 0xd6, 0xdc, 0x83, 0xe1, 0x89, 0x2a, 0xa2, 0xf4, 0x72, 0x19, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29, 0xf4, 0xb8, 0x48, 0x95, 0x96, 0x6d, 0xb2, 0xb1, 0xc6, 0xe8, 0x71, 0x9e, 0x65, 0xb1, 0x16, 0xbb, 0x1e, 0x0b, 0x1c, 0xd1, 0x10, 0x7c, 0x02, 0x70, 0x18, 0x67, 0x61, 0x7d, 0xb6, 0xe7, 0xb1, 0x80, 0x89, 0x16, 0xe3, 0xef, 0x42, 0x1f, 0x6f, 0xfa, 0x32, 0xcc, 0x1b, 0xb7, 0xec, 0x0f, 0x6e, 0xfd, 0xcf, 0x0c, 0x46, 0xaf, 0x2b, 0x59, 0xac, 0x84, 0x7c, 0x5f, 0xc9, 0x92, 0xba, 0x42, 0xb8, 0x76, - 0xa9, 0x01, 0xdf, 0x81, 0xde, 0x32, 0x8e, 0xce, 0xa5, 0x7e, 0x3b, 0x5b, 0xd4, 0x08, 0xbd, 0x36, - 0x6f, 0x5e, 0x92, 0x57, 0x47, 0xb4, 0x29, 0x3c, 0x29, 0x64, 0x92, 0x29, 0x63, 0xa6, 0x46, 0xdc, - 0x87, 0xd1, 0xc1, 0xf5, 0x79, 0x5c, 0x5d, 0x48, 0x7d, 0xb4, 0x47, 0xea, 0x06, 0x87, 0xd5, 0x6b, - 0x4c, 0x89, 0xef, 0xeb, 0xea, 0x2d, 0xca, 0xff, 0xc0, 0x60, 0x5c, 0x5f, 0xbf, 0xcc, 0xb3, 0xb4, - 0x94, 0xd8, 0xa3, 0x83, 0xa2, 0x30, 0x3d, 0x3a, 0x28, 0x0a, 0xbe, 0x0b, 0x7d, 0x21, 0xcb, 0x2a, - 0x56, 0xa6, 0xf1, 0x0f, 0x9a, 0xa7, 0x30, 0x67, 0xab, 0x58, 0x09, 0xb3, 0x8b, 0x3f, 0x87, 0xad, - 0x8d, 0x20, 0xe9, 0x89, 0x19, 0xce, 0x1e, 0x36, 0xe7, 0x36, 0x74, 0x71, 0x6b, 0xbb, 0xff, 0x8d, - 0xc1, 0xb0, 0x55, 0x99, 0x07, 0x66, 0x78, 0xe9, 0x5a, 0xc3, 0xd9, 0x76, 0x53, 0x48, 0xf3, 0xc2, - 0x0c, 0xf7, 0x08, 0xd8, 0x71, 0x1d, 0x26, 0x76, 0x8c, 0x2d, 0xc4, 0xe1, 0x34, 0xdf, 0x6f, 0xb5, - 0x10, 0x69, 0xa1, 0x45, 0xee, 0x42, 0x7f, 0xef, 0x5d, 0x98, 0x5e, 0xca, 0x0b, 0x0a, 0x93, 0x23, - 0x0c, 0xe4, 0xd3, 0x66, 0x38, 0xe9, 0xf5, 0x87, 0x33, 0xde, 0x94, 0x30, 0x8a, 0x68, 0x06, 0xd8, - 0xa4, 0x19, 0x7b, 0x31, 0xd6, 0x69, 0xf6, 0x7f, 0x32, 0x18, 0x2f, 0x92, 0x3c, 0x2b, 0x54, 0x2b, - 0x21, 0x8b, 0xf4, 0x42, 0x5e, 0x9b, 0x84, 0x10, 0x40, 0xf6, 0xb0, 0x08, 0x13, 0x3d, 0x0a, 0x03, - 0xa1, 0x01, 0xb2, 0x94, 0x14, 0x4a, 0x86, 0x2d, 0x34, 0xa0, 0x4c, 0xe0, 0xb0, 0x97, 0xae, 0xad, - 0xd3, 0xa4, 0x11, 0x66, 0xdf, 0xcc, 0x7a, 0xe9, 0x76, 0x49, 0x6a, 0x08, 0xcc, 0xfe, 0x7a, 0xd8, - 0x31, 0x2f, 0x56, 0x60, 0x89, 0x16, 0x83, 0xef, 0x20, 0xb2, 0x2b, 0xfa, 0x85, 0xeb, 0xd3, 0x2f, - 0x9c, 0x81, 0x78, 0x52, 0x97, 0x21, 0xd1, 0x21, 0xb1, 0xc5, 0xf8, 0x5f, 0x18, 0x70, 0xed, 0x91, - 0xa6, 0xe8, 0xff, 0x19, 0xc5, 0xbd, 0x91, 0x8c, 0x75, 0x63, 0x70, 0x2f, 0x82, 0xbf, 0xd8, 0xdc, - 0x81, 0x1e, 0xdd, 0xc2, 0x58, 0xac, 0xd1, 0x2d, 0x13, 0xfd, 0xdb, 0x26, 0xe6, 0xdb, 0x5f, 0x6f, - 0x26, 0xec, 0xfb, 0xcd, 0x84, 0xfd, 0xb8, 0x99, 0xb0, 0x8f, 0xbf, 0x26, 0xf7, 0xce, 0x7a, 0xf4, - 0x27, 0xf2, 0xf4, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa3, 0xa0, 0xd2, 0x51, 0x54, 0x06, 0x00, + 0xa9, 0x01, 0xdf, 0x81, 0xde, 0x49, 0x1c, 0x9d, 0x4b, 0xfd, 0x76, 0xb6, 0xa8, 0x11, 0x7a, 0x6d, + 0xde, 0xbc, 0x24, 0xaf, 0x8e, 0x68, 0x53, 0x78, 0x52, 0xc8, 0x24, 0x53, 0xc6, 0x4c, 0x8d, 0xb8, + 0x0f, 0xa3, 0x83, 0xeb, 0xf3, 0xb8, 0xba, 0x90, 0xfa, 0x68, 0x8f, 0xd4, 0x0d, 0x0e, 0xab, 0xd7, + 0x98, 0x12, 0xdf, 0xd7, 0xd5, 0x5b, 0x94, 0xff, 0x81, 0xc1, 0xb8, 0xbe, 0x7e, 0x99, 0x67, 0x69, + 0x29, 0xb1, 0x47, 0x07, 0x45, 0x61, 0x7a, 0x74, 0x50, 0x14, 0x7c, 0x17, 0xfa, 0x42, 0x96, 0x55, + 0xac, 0x4c, 0xe3, 0x1f, 0x34, 0x4f, 0x61, 0xce, 0x56, 0xb1, 0x12, 0x66, 0x17, 0x7f, 0x0e, 0x5b, + 0x1b, 0x41, 0xd2, 0x13, 0x33, 0x9c, 0x3d, 0x6c, 0xce, 0x6d, 0xe8, 0xe2, 0xd6, 0x76, 0xff, 0x1b, + 0x83, 0x61, 0xab, 0x32, 0x0f, 0xcc, 0xf0, 0xd2, 0xb5, 0x86, 0xb3, 0xed, 0xa6, 0x90, 0xe6, 0x85, + 0x19, 0xee, 0x11, 0xb0, 0xe3, 0x3a, 0x4c, 0xec, 0x18, 0x5b, 0x88, 0xc3, 0x69, 0xbe, 0xdf, 0x6a, + 0x21, 0xd2, 0x42, 0x8b, 0xdc, 0x85, 0xfe, 0xde, 0xbb, 0x30, 0xbd, 0x94, 0x17, 0x14, 0x26, 0x47, + 0x18, 0xc8, 0xa7, 0xcd, 0x70, 0xd2, 0xeb, 0x0f, 0x67, 0xbc, 0x29, 0x61, 0x14, 0xd1, 0x0c, 0xb0, + 0x49, 0x33, 0xf6, 0x62, 0xac, 0xd3, 0xec, 0xff, 0x64, 0x30, 0x5e, 0x24, 0x79, 0x56, 0xa8, 0x56, + 0x42, 0x16, 0xe9, 0x85, 0xbc, 0x36, 0x09, 0x21, 0x80, 0xec, 0x61, 0x11, 0x26, 0x7a, 0x14, 0x06, + 0x42, 0x03, 0x64, 0x29, 0x29, 0x94, 0x0c, 0x5b, 0x68, 0x40, 0x99, 0xc0, 0x61, 0x2f, 0x5d, 0x5b, + 0xa7, 0x49, 0x23, 0xcc, 0xbe, 0x99, 0xf5, 0xd2, 0xed, 0x92, 0xd4, 0x10, 0x98, 0xfd, 0xf5, 0xb0, + 0x63, 0x5e, 0xac, 0xc0, 0x12, 0x2d, 0x06, 0xdf, 0x41, 0x64, 0x57, 0xf4, 0x0b, 0xd7, 0xa7, 0x5f, + 0x38, 0x03, 0xf1, 0xa4, 0x2e, 0x43, 0xa2, 0x43, 0x62, 0x8b, 0xf1, 0xbf, 0x30, 0xe0, 0xda, 0x23, + 0x4d, 0xd1, 0xff, 0x33, 0x8a, 0x7b, 0x23, 0x19, 0xeb, 0xc6, 0xe0, 0x5e, 0x04, 0x7f, 0xb1, 0xb9, + 0x03, 0x3d, 0xba, 0x85, 0xb1, 0x58, 0xa3, 0x5b, 0x26, 0xfa, 0xb7, 0x4d, 0xcc, 0xb7, 0xbf, 0xde, + 0x4c, 0xd8, 0xf7, 0x9b, 0x09, 0xfb, 0x71, 0x33, 0x61, 0x1f, 0x7f, 0x4d, 0xee, 0x9d, 0xf5, 0xe8, + 0x4f, 0xe4, 0xe9, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x3e, 0xba, 0x23, 0x54, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 5fa40cfab..026a3748b 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -14,8 +14,8 @@ message Pair { uint64 Count = 2; } -message SumCount { - int64 Sum = 1; +message ValCount { + int64 Val = 1; int64 Count = 2; } @@ -64,7 +64,7 @@ message QueryResult { Bitmap Bitmap = 1; uint64 N = 2; repeated Pair Pairs = 3; - SumCount SumCount = 5; + ValCount ValCount = 5; bool Changed = 4; } diff --git a/view.go b/view.go index f775f1b0e..dbcc73b8d 100644 --- a/view.go +++ b/view.go @@ -357,6 +357,49 @@ func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err e return sum, count, nil } +// FieldMin returns the min and count of a field. +func (v *View) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) { + var minHasValue bool + for _, f := range v.Fragments() { + fmin, fcount, err := f.FieldMin(filter, bitDepth) + if err != nil { + return min, count, err + } + // Don't consider a min based on zero columns. + if fcount == 0 { + continue + } + + if !minHasValue { + min = fmin + minHasValue = true + count += fcount + continue + } + + if fmin < min { + min = fmin + count += fcount + } + } + return min, count, nil +} + +// FieldMax returns the max and count of a field. +func (v *View) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) { + for _, f := range v.Fragments() { + fmax, fcount, err := f.FieldMax(filter, bitDepth) + if err != nil { + return max, count, err + } + if fcount > 0 && fmax > max { + max = fmax + count += fcount + } + } + return max, count, nil +} + // FieldRange returns bitmaps with a field value encoding matching the predicate. func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { bm := NewBitmap()