scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-10 04:47:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into distinct-failure

Browse source
This commit is contained in:
Cody Soyland 2021-01-19 16:50:25 -06:00 • committed by GitHub
commit 0a325dcdb8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
45 changed files with 3015 additions and 872 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Makefile
View file

@ -152,6 +152,9 @@ prerelease-upload:
install:
go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
install-bench:
go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench
lattice:
git clone git@github.com:molecula/lattice.git

4
api.go
View file

@ -163,13 +163,15 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
if !req.Remote {
defer api.tracker.Finish(api.tracker.Start(req.Query, api.server.nodeID, req.Index, start))
}
// TODO can we get rid of exec options and pass the QueryRequest directly to executor?
execOpts := &execOptions{
Remote: req.Remote,
Profile: req.Profile,
ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat.
ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat.
ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat.
EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request
PreTranslated: req.PreTranslated,
EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request
}
resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
if err != nil {

2
cluster.go
View file

@ -1047,7 +1047,7 @@ func (c *cluster) ShardNodes(index string, shard uint64) []*Node {
return c.shardNodes(index, shard)
}
// shardNodes returns a list of nodes that own a fragment. unprotected
// shardNodes returns a list of nodes that own a shard. unprotected
func (c *cluster) shardNodes(index string, shard uint64) []*Node {
return c.partitionNodes(c.shardToShardPartition(index, shard))
}

313
cmd/pilosa-bench/main.go Normal file
View file

@ -0,0 +1,313 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"flag"
"fmt"
"io/ioutil"
"log"
"math/rand"
"net/http"
"os"
"sort"
"strings"
"time"
"github.com/pilosa/pilosa/v2"
phttp "github.com/pilosa/pilosa/v2/http"
"golang.org/x/sync/errgroup"
)
func main() {
if err := run(context.Background(), os.Args[1:]); err == flag.ErrHelp {
os.Exit(1)
} else if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run(ctx context.Context, args []string) (err error) {
fs := flag.NewFlagSet("pilosa-bench", flag.ContinueOnError)
hostport := fs.String("hostport", "localhost:10101", "")
typ := fs.String("type", "row", "query type (row)")
n := fs.Int("n", 1000, "number of queries")
rate := fs.Int("rate", 1, "number of queries per second")
verbose := fs.Bool("v", false, "verbose logging")
from := fs.String("from", "", "from time for row-range queries (ISO 8601)")
to := fs.String("to", "", "to time for row-range queries (ISO 8601)")
if err := fs.Parse(args); err != nil {
return err
}
// Parse from/to time.
var opt queryOptions
if *from != "" {
if opt.from, err = time.Parse(time.RFC3339, *from); err != nil {
return fmt.Errorf("cannot parse -from time")
}
}
if *to != "" {
if opt.to, err = time.Parse(time.RFC3339, *to); err != nil {
return fmt.Errorf("cannot parse -to time")
}
}
if (*typ == "row-range" || *typ == "topk") && (opt.from.IsZero() || opt.to.IsZero()) {
return fmt.Errorf("-from and -to flags must be specified for topk & row-range queries")
}
// Clear time prefix on log.
log.SetFlags(0)
if !*verbose {
log.SetOutput(ioutil.Discard)
}
// Setup PRNG to have consistent values for the same set of data.
rand.Seed(0)
// Setup connection to pilosa.
client, err := phttp.NewInternalClient(*hostport, http.DefaultClient)
if err != nil {
return err
}
// Load all id/keys for each field.
log.Printf("loading field identifiers")
fieldIDMap, err := loadFields(ctx, client)
if err != nil {
return fmt.Errorf("cannot load field identifiers: %w", err)
} else if len(fieldIDMap) == 0 {
return fmt.Errorf("no field identifiers available, please verify data exists")
}
// Generate list of sorted keys.
fieldKeys := make([]fieldKey, 0, len(fieldIDMap))
for k, f := range fieldIDMap {
switch *typ {
case "row-bsi":
if f.info.Options.Type != "int" {
continue
}
case "row-range", "topk":
if f.info.Options.Type != "time" {
continue
}
default:
if f.info.Options.Type == "int" || f.info.Options.Type == "time" {
continue
}
}
fieldKeys = append(fieldKeys, k)
}
sort.Slice(fieldKeys, func(i, j int) bool {
return compareFieldKeys(fieldKeys[i], fieldKeys[j]) == -1
})
// Ensure we have appropriate fields for our query type.
if len(fieldKeys) == 0 {
return fmt.Errorf("no available fields are appropriate for %q queries", *typ)
}
log.Printf("issuing %d queries at %d query/sec", *n, *rate)
// Repeatedly issue queries based on available row data.
ticker := time.NewTicker(time.Second / time.Duration(*rate))
var g errgroup.Group
for i := 0; i < *n; i++ {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
key := fieldKeys[rand.Intn(len(fieldKeys))]
q, err := generateQuery(*typ, key.index, key.field, fieldIDMap[key].info, fieldIDMap[key].identifiers, opt)
if err != nil {
return fmt.Errorf("cannot generate query: %w", err)
}
log.Printf("[query] %s", q)
g.Go(func() error {
_, err = client.Query(ctx, key.index, &pilosa.QueryRequest{Index: key.index, Query: q})
if err != nil {
return err
}
return nil
})
}
return g.Wait()
}
func generateQuery(typ, index, field string, info *pilosa.FieldInfo, identifiers *pilosa.RowIdentifiers, opt queryOptions) (string, error) {
switch typ {
case "row":
return generateRowQuery(index, field, identifiers), nil
case "row-bsi":
return generateRowBSIQuery(index, field), nil
case "row-range":
return generateRowRangeQuery(index, field, identifiers, opt.from, opt.to), nil
case "count":
return generateCountQuery(index, field, identifiers), nil
case "intersect":
return generateIntersectQuery(index, field, identifiers), nil
case "union":
return generateUnionQuery(index, field, identifiers), nil
case "difference":
return generateDifferenceQuery(index, field, identifiers), nil
case "xor":
return generateXorQuery(index, field, identifiers), nil
case "groupby":
return generateGroupByQuery(index, field), nil
case "topk":
return generateTopKQuery(index, field, opt.from, opt.to), nil
default:
return "", fmt.Errorf("invalid query type: %q", typ)
}
}
func generateRowQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
if len(identifiers.Rows) > 0 {
return fmt.Sprintf("Row(%s=%d)", field, chooseRowID(identifiers))
}
return fmt.Sprintf("Row(%s=%q)", field, chooseRowKey(identifiers))
}
func generateRowBSIQuery(index, field string) string {
return fmt.Sprintf("Row(%s > 0)", field)
}
func generateRowRangeQuery(index, field string, identifiers *pilosa.RowIdentifiers, from, to time.Time) string {
if len(identifiers.Rows) > 0 {
return fmt.Sprintf("Row(%s=%d, from='%s', to='%s')", field, chooseRowID(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
}
return fmt.Sprintf("Row(%s=%q, from='%s', to='%s')", field, chooseRowKey(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
}
func generateRowQueries(index, field string, identifiers *pilosa.RowIdentifiers) string {
a := make([]string, rand.Intn(9)+1)
for i := range a {
a[i] = generateRowQuery(index, field, identifiers)
}
return strings.Join(a, ", ")
}
func generateCountQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
return fmt.Sprintf("Count(%s)", generateRowQuery(index, field, identifiers))
}
func generateIntersectQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
return fmt.Sprintf("Intersect(%s)", generateRowQueries(index, field, identifiers))
}
func generateUnionQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
return fmt.Sprintf("Union(%s)", generateRowQueries(index, field, identifiers))
}
func generateDifferenceQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
return fmt.Sprintf("Difference(%s)", generateRowQueries(index, field, identifiers))
}
func generateXorQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
return fmt.Sprintf("Xor(%s)", generateRowQueries(index, field, identifiers))
}
func generateGroupByQuery(index, field string) string {
return fmt.Sprintf("GroupBy(Rows(%s))", field)
}
func generateTopKQuery(index, field string, from, to time.Time) string {
return fmt.Sprintf("TopK(%s, from='%s', to='%s')", field, from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
}
// loadFields returns a mapping of index/field names to field info & identifiers.
func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey]*fieldInfo, error) {
indexes, err := client.Schema(ctx)
if err != nil {
return nil, err
}
m := make(map[fieldKey]*fieldInfo)
for _, ii := range indexes {
for _, f := range ii.Fields {
log.Printf("field: index=%s name=%s type=%s", ii.Name, f.Name, f.Options.Type)
switch f.Options.Type {
case "set", "mutex", "time":
identifiers, err := fetchFieldIDs(ctx, client, ii.Name, f.Name)
if err != nil {
return nil, fmt.Errorf("fetch fields: %w", err)
} else if len(identifiers.Rows) > 0 || len(identifiers.Keys) > 0 {
m[fieldKey{ii.Name, f.Name}] = &fieldInfo{f, identifiers}
}
case "int":
m[fieldKey{ii.Name, f.Name}] = &fieldInfo{info: f}
}
}
}
return m, nil
}
// fetchFieldIDs returns a list of field IDs or keys.
func fetchFieldIDs(ctx context.Context, client *phttp.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) {
resp, err := client.Query(ctx, indexName, &pilosa.QueryRequest{Index: indexName, Query: `Rows(` + fieldName + `)`})
if err != nil {
return nil, err
}
switch result := resp.Results[0].(type) {
case *pilosa.RowIdentifiers:
return result, nil
case pilosa.RowIdentifiers:
return &result, nil
default:
return nil, fmt.Errorf("unexpected result type: %T", result)
}
}
func chooseRowID(identifiers *pilosa.RowIdentifiers) uint64 {
return identifiers.Rows[rand.Intn(len(identifiers.Rows))]
}
func chooseRowKey(identifiers *pilosa.RowIdentifiers) string {
return identifiers.Keys[rand.Intn(len(identifiers.Keys))]
}
type fieldKey struct {
index string
field string
}
type fieldInfo struct {
info *pilosa.FieldInfo
identifiers *pilosa.RowIdentifiers
}
func compareFieldKeys(x, y fieldKey) int {
if cmp := strings.Compare(x.index, y.index); cmp != 0 {
return cmp
}
return strings.Compare(x.field, y.field)
}
type queryOptions struct {
from, to time.Time
}

4
cmd/root_test.go
View file

@ -148,6 +148,10 @@ func (ct *commandTest) setupCommand(t *testing.T) *cobra.Command {
err = os.Setenv(name, val)
failErr(t, err, fmt.Sprintf("setting environment variable '%s' to '%s'", name, val))
}
// address common case where system might have postgres bind
// setting that an existing running Pilosa is using (causing test
// failures due to port conflict)
os.Setenv("PILOSA_POSTGRES_BIND", "")
// make command and set args
rc := cmd.NewRootCommand(strings.NewReader(""), ioutil.Discard, ioutil.Discard)

130
cmd/server_test.go
View file

@ -15,6 +15,7 @@
package cmd_test
import (
"fmt"
"io/ioutil"
"strings"
"testing"
@ -169,38 +170,49 @@ func TestServerConfig(t *testing.T) {
// run server tests
for i, test := range tests {
com := test.setupCommand(t)
executed := make(chan struct{})
var execErr error
go func() {
execErr = com.Execute()
close(executed)
}()
select {
case <-cmd.Server.Started:
case <-executed:
}
if execErr != nil {
t.Fatalf("executing server command: %v", execErr)
}
err := cmd.Server.Close()
failErr(t, err, "closing pilosa server command")
<-executed
failErr(t, execErr, "executing command")
t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
com := test.setupCommand(t)
executed := make(chan struct{})
var execErr error
go func() {
execErr = com.Execute()
close(executed)
}()
select {
case <-cmd.Server.Started:
case <-executed:
}
if execErr != nil {
t.Fatalf("executing server command: %v", execErr)
}
err := cmd.Server.Close()
failErr(t, err, "closing pilosa server command")
<-executed
failErr(t, execErr, "executing command")
if err := test.validation(); err != nil {
t.Fatalf("Failed test %d due to: %v", i, err)
}
test.reset()
if err := test.validation(); err != nil {
t.Fatalf("Failed test %d due to: %v", i, err)
}
test.reset()
})
}
}
func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
actualDataDir, err := ioutil.TempDir("", "")
failErr(t, err, "making data dir")
tests := []commandTest{
// TEST 0
{
args: []string{"server", "--long-query-time", "1m10s"},
env: map[string]string{},
cfgFileContent: "",
args: []string{"server", "--long-query-time", "1m10s"},
env: map[string]string{},
cfgFileContent: `
bind = "localhost:0"
bind-grpc = "localhost:0"
data-dir = "` + actualDataDir + `"
[gossip]
port = "14321"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*70))
@ -210,9 +222,14 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
},
// TEST 1
{
args: []string{"server", "--cluster.long-query-time", "1m20s"},
env: map[string]string{},
cfgFileContent: "",
args: []string{"server", "--cluster.long-query-time", "1m20s"},
env: map[string]string{},
cfgFileContent: `
bind = "localhost:0"
bind-grpc = "localhost:0"
[gossip]
port = "14321"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*80))
@ -222,9 +239,14 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
},
// TEST 2: Use old value if both are provided because it is the simplest implementation
{
args: []string{"server", "--long-query-time", "50s", "--cluster.long-query-time", "1m30s"},
env: map[string]string{},
cfgFileContent: "",
args: []string{"server", "--long-query-time", "50s", "--cluster.long-query-time", "1m30s"},
env: map[string]string{},
cfgFileContent: `
bind = "localhost:0"
bind-grpc = "localhost:0"
[gossip]
port = "14321"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*50))
@ -237,28 +259,30 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
// run server tests
for i, test := range tests {
com := test.setupCommand(t)
executed := make(chan struct{})
var execErr error
go func() {
execErr = com.Execute()
close(executed)
}()
select {
case <-cmd.Server.Started:
case <-executed:
}
if execErr != nil {
t.Fatalf("executing server command: %v", execErr)
}
err := cmd.Server.Close()
failErr(t, err, "closing pilosa server command")
<-executed
failErr(t, execErr, "executing command")
t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
com := test.setupCommand(t)
executed := make(chan struct{})
var execErr error
go func() {
execErr = com.Execute()
close(executed)
}()
select {
case <-cmd.Server.Started:
case <-executed:
}
if execErr != nil {
t.Fatalf("executing server command: %v", execErr)
}
err := cmd.Server.Close()
failErr(t, err, "closing pilosa server command")
<-executed
failErr(t, execErr, "executing command")
if err := test.validation(); err != nil {
t.Fatalf("Failed test %d due to: %v", i, err)
}
test.reset()
if err := test.validation(); err != nil {
t.Fatalf("Failed test %d due to: %v", i, err)
}
test.reset()
})
}
}

46
encoding/proto/proto.go
View file

@ -490,6 +490,7 @@ func (s Serializer) encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRe
Remote: m.Remote,
ExcludeRowAttrs: m.ExcludeRowAttrs,
ExcludeColumns: m.ExcludeColumns,
PreTranslated: m.PreTranslated,
EmbeddedData: make([]*internal.Row, len(m.EmbeddedData)),
}
for i := range m.EmbeddedData {
@ -535,7 +536,7 @@ func (s Serializer) encodeQueryResponse(m *pilosa.QueryResponse) *internal.Query
case pilosa.ExtractedIDMatrix:
pb.Results[i].Type = queryResultTypeExtractedIDMatrix
pb.Results[i].ExtractedIDMatrix = s.endcodeExtractedIDMatrix(result)
case []pilosa.GroupCount:
case *pilosa.GroupCounts:
pb.Results[i].Type = queryResultTypeGroupCounts
pb.Results[i].GroupCounts = s.encodeGroupCounts(result)
case pilosa.RowIdentifiers:
@ -1210,6 +1211,7 @@ func (s Serializer) decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.Quer
m.ExcludeRowAttrs = pb.ExcludeRowAttrs
m.ExcludeColumns = pb.ExcludeColumns
m.EmbeddedData = make([]*pilosa.Row, len(pb.EmbeddedData))
m.PreTranslated = pb.PreTranslated
for i := range pb.EmbeddedData {
m.EmbeddedData[i] = s.decodeRow(pb.EmbeddedData[i])
}
@ -1421,7 +1423,7 @@ func (s Serializer) decodeQueryResult(pb *internal.QueryResult) interface{} {
case queryResultTypeRowIdentifiers:
return s.decodeRowIdentifiers(pb.RowIdentifiers)
case queryResultTypeGroupCounts:
return s.decodeGroupCounts(pb.GroupCounts)
return s.decodeGroupCounts(pb.GroupCounts, pb.OldGroupCounts)
case queryResultTypePair:
return s.decodePair(pb.Pairs[0])
case queryResultTypePairField:
@ -1585,16 +1587,22 @@ func (s Serializer) decodeRowIdentifiers(a *internal.RowIdentifiers) *pilosa.Row
}
}
func (s Serializer) decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount {
other := make([]pilosa.GroupCount, len(a))
for i := range a {
func (s Serializer) decodeGroupCounts(a *internal.GroupCounts, b []*internal.GroupCount) *pilosa.GroupCounts {
// Workaround: If we get an old-style "[]*GroupCount", we translate it.
if a == nil {
a = &internal.GroupCounts{Aggregate: "", Groups: b}
}
other := make([]pilosa.GroupCount, len(a.Groups))
for i, gc := range a.Groups {
other[i] = pilosa.GroupCount{
Group: s.decodeFieldRows(a[i].Group),
Count: a[i].Count,
Sum: a[i].Sum,
Group: s.decodeFieldRows(gc.Group),
Count: gc.Count,
// note: not renaming the `internal` structure members now
// to avoid breaking protobuf interactions.
Agg: gc.Agg,
}
}
return other
return pilosa.NewGroupCounts(a.Aggregate, other...)
}
func (s Serializer) decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow {
@ -1696,7 +1704,7 @@ func (s Serializer) encodeSignedRow(r pilosa.SignedRow) *internal.SignedRow {
func (s Serializer) encodeRow(r *pilosa.Row) *internal.Row {
if r == nil {
return nil
return &internal.Row{} // Generated proto code doesn't like a nil Row.
}
ir := &internal.Row{
@ -1721,13 +1729,17 @@ func (s Serializer) encodeRowIdentifiers(r pilosa.RowIdentifiers) *internal.RowI
}
}
func (s Serializer) encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount {
result := make([]*internal.GroupCount, len(counts))
for i := range counts {
result[i] = &internal.GroupCount{
Group: s.encodeFieldRows(counts[i].Group),
Count: counts[i].Count,
Sum: counts[i].Sum,
func (s Serializer) encodeGroupCounts(counts *pilosa.GroupCounts) *internal.GroupCounts {
groups := counts.Groups()
result := &internal.GroupCounts{
Groups: make([]*internal.GroupCount, len(groups)),
Aggregate: counts.AggregateColumn(),
}
for i, gc := range groups {
result.Groups[i] = &internal.GroupCount{
Group: s.encodeFieldRows(gc.Group),
Count: gc.Count,
Agg: gc.Agg,
}
}
return result

506
executor.go
View file

@ -24,6 +24,7 @@ import (
"strings"
"sync"
"time"
"unsafe"
"github.com/pilosa/pilosa/v2/pql"
pb "github.com/pilosa/pilosa/v2/proto"
@ -312,6 +313,8 @@ func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) {
out.Results = append(out.Results, x)
case []GroupCount:
out.Results = append(out.Results, x)
case *GroupCounts:
out.Results = append(out.Results, x)
case ExtractedTable:
out.Results = append(out.Results, x)
case ExtractedIDMatrix:
@ -441,9 +444,11 @@ func (e *executor) handlePreCalls(ctx context.Context, qcx *Qcx, index string, c
// shards if the query has to go to them
opt.EmbeddedData = append(opt.EmbeddedData, row)
// and stash a copy locally, so local calls can use it
c.Precomputed = make(map[uint64]interface{}, len(row.segments))
for _, segment := range row.segments {
c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}}
if row != nil {
c.Precomputed = make(map[uint64]interface{}, len(row.segments))
for _, segment := range row.segments {
c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}}
}
}
return nil
}
@ -468,7 +473,11 @@ func (e *executor) handlePreCallChildren(ctx context.Context, qcx *Qcx, index st
return err
}
}
for _, val := range c.Args {
for key, val := range c.Args {
// Do not precompute GroupBy aggregates
if key == "aggregate" {
continue
}
// Handle Call() operations which exist inside named arguments, too.
if call, ok := val.(*pql.Call); ok {
if err := ctx.Err(); err != nil {
@ -528,7 +537,7 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q
}
// Apply call translation.
if !opt.Remote {
if !opt.Remote && !opt.PreTranslated {
translated, err := e.translateCall(call, index, colTranslations, rowTranslations)
if err != nil {
return nil, errors.Wrap(err, "translating call")
@ -672,77 +681,101 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
switch c.Name {
case "Sum":
statFn()
return e.executeSum(ctx, qcx, index, c, shards, opt)
res, err := e.executeSum(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeSum %v", shards)
case "Min":
statFn()
return e.executeMin(ctx, qcx, index, c, shards, opt)
res, err := e.executeMin(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeMin %v", shards)
case "Max":
statFn()
return e.executeMax(ctx, qcx, index, c, shards, opt)
res, err := e.executeMax(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeMax %v", shards)
case "MinRow":
statFn()
return e.executeMinRow(ctx, qcx, index, c, shards, opt)
res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeMinRow %v", shards)
case "MaxRow":
statFn()
return e.executeMaxRow(ctx, qcx, index, c, shards, opt)
res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeMaxRow %v", shards)
case "Clear":
statFn()
return e.executeClearBit(ctx, qcx, index, c, opt)
res, err := e.executeClearBit(ctx, qcx, index, c, opt)
return res, errors.Wrapf(err, "executeClearBit %v", shards)
case "ClearRow":
statFn()
return e.executeClearRow(ctx, qcx, index, c, shards, opt)
res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeClearRow %v", shards)
case "Distinct":
statFn()
return e.executeDistinct(ctx, qcx, index, c, shards, opt)
res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeDistinct %v", shards)
case "Store":
statFn()
return e.executeSetRow(ctx, qcx, index, c, shards, opt)
res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeSetRow %v", shards)
case "Count":
statFn()
return e.executeCount(ctx, qcx, index, c, shards, opt)
res, err := e.executeCount(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeCount %v", shards)
case "Set":
statFn()
return e.executeSet(ctx, qcx, index, c, opt)
res, err := e.executeSet(ctx, qcx, index, c, opt)
return res, errors.Wrapf(err, "executeSet %v", shards)
case "SetRowAttrs":
statFn()
return nil, e.executeSetRowAttrs(ctx, qcx, index, c, opt)
return nil, errors.Wrap(e.executeSetRowAttrs(ctx, qcx, index, c, opt), "executeSetRowAttrs")
case "SetColumnAttrs":
statFn()
return nil, e.executeSetColumnAttrs(ctx, qcx, index, c, opt)
return nil, errors.Wrap(e.executeSetColumnAttrs(ctx, qcx, index, c, opt), "executeSetColumnAttrs")
case "TopK":
statFn()
return e.executeTopK(ctx, qcx, index, c, shards, opt)
res, err := e.executeTopK(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeTopK %v", shards)
case "TopN":
statFn()
return e.executeTopN(ctx, qcx, index, c, shards, opt)
res, err := e.executeTopN(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeTopN %v", shards)
case "Rows":
statFn()
return e.executeRows(ctx, qcx, index, c, shards, opt)
res, err := e.executeRows(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeRows %v", shards)
case "Extract":
statFn()
return e.executeExtract(ctx, qcx, index, c, shards, opt)
res, err := e.executeExtract(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeExtract %v", shards)
case "GroupBy":
statFn()
return e.executeGroupBy(ctx, qcx, index, c, shards, opt)
res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeGroupBy %v", shards)
case "Options":
statFn()
return e.executeOptionsCall(ctx, qcx, index, c, shards, opt)
res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeOptionsCall %v", shards)
case "IncludesColumn":
return e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt)
res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shards)
case "FieldValue":
statFn()
return e.executeFieldValueCall(ctx, qcx, index, c, shards, opt)
res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeFieldValueCall %v", shards)
case "Precomputed":
return e.executePrecomputedCall(ctx, qcx, index, c, shards, opt)
res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executePrecomputedCall %v", shards)
case "UnionRows":
return e.executeUnionRows(ctx, qcx, index, c, shards, opt)
res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeUnionRows %v", shards)
case "ConstRow":
return e.executeConstRow(ctx, index, c)
res, err := e.executeConstRow(ctx, index, c)
return res, errors.Wrapf(err, "executeConstRow %v", shards)
case "Limit":
return e.executeLimitCall(ctx, qcx, index, c, shards, opt)
res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeLimitCall %v", shards)
default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap.
statFn()
return e.executeBitmapCall(ctx, qcx, index, c, shards, opt)
res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeBitmapCall %v", shards)
}
}
@ -1107,6 +1140,11 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string,
case SignedRow:
return other.union(v.(SignedRow))
case *Row:
if other == nil {
return v
} else if v.(*Row) == nil {
return other
}
return other.Union(v.(*Row))
case nil:
return v
@ -1117,13 +1155,12 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string,
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "mapReduce")
}
if other, ok := result.(SignedRow); ok {
other.field = field
}
return result, nil
}
@ -1466,10 +1503,15 @@ func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldNam
defer finisher(&err0)
fragData, _, err := tx.ContainerIterator(index, fieldName, "standard", shard, 0)
if err != nil {
switch errors.Cause(err) {
case ViewNotFound, FragmentNotFound:
return nil, nil
case nil:
default:
return nil, errors.Wrap(err, "getting fragment data")
}
defer fragData.Close()
// We can't grab the containers "for each row" from the set-type field,
// because we don't know how many rows there are, and some of them
// might be empty, so really, we're going to iterate through the
@ -2576,7 +2618,106 @@ func (r RowIDs) merge(other RowIDs, limit int) RowIDs {
return result
}
func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]GroupCount, error) {
// order denotes sort order—can be asc or desc (see constants below).
type order bool
const (
asc order = true
desc order = false
)
// groupCountSorter sorts the output of a GroupBy request (a
// []GroupCount) according to sorting instructions encoded in "fields"
// and "order".
//
// Each field in "fields" is an integer which can be -1 to denote
// sorting on the Count and -2 to denote sorting on the
// sum/aggregate. Currently nothing else is supported, but the idea
// was that if there were positive integers they would be indexes into
// GroupCount.FieldRow and allowing sorting on the values of different
// fields in the group. Each item in "order" corresponds to the same
// index in "fields" and denotes the order of the sort.
type groupCountSorter struct {
fields []int
order []order
data []GroupCount
}
func (g *groupCountSorter) Len() int { return len(g.data) }
func (g *groupCountSorter) Swap(i, j int) { g.data[i], g.data[j] = g.data[j], g.data[i] }
func (g *groupCountSorter) Less(i, j int) bool {
gci, gcj := g.data[i], g.data[j]
for idx, fieldIndex := range g.fields {
fieldOrder := g.order[idx]
switch fieldIndex {
case -1: // Count
if gci.Count < gcj.Count {
return fieldOrder == asc
} else if gci.Count > gcj.Count {
return fieldOrder == desc
}
case -2: // Aggregate
if gci.Agg < gcj.Agg {
return fieldOrder == asc
} else if gci.Agg > gcj.Agg {
return fieldOrder == desc
}
default:
panic("impossible")
}
}
return false
}
// getSorter hackily parses the sortSpec and figures out how to sort
// the GroupBy results.
func getSorter(sortSpec string) (*groupCountSorter, error) {
gcs := &groupCountSorter{
fields: []int{},
order: []order{},
}
sortOn := strings.Split(sortSpec, ",")
for _, sortField := range sortOn {
sortField = strings.TrimSpace(sortField)
fieldDir := strings.Fields(sortField)
if len(fieldDir) == 0 {
return nil, errors.Errorf("invalid sorting directive: '%s'", sortField)
} else if fieldDir[0] == "count" {
gcs.fields = append(gcs.fields, -1)
} else if fieldDir[0] == "aggregate" || fieldDir[0] == "sum" {
gcs.fields = append(gcs.fields, -2)
} else {
return nil, errors.Errorf("sorting is only supported on count, aggregate, or sum, not '%s'", fieldDir[0])
}
if len(fieldDir) == 1 {
gcs.order = append(gcs.order, desc)
} else if len(fieldDir) > 2 {
return nil, errors.Errorf("parsing sort directive: '%s': too many elements", sortField)
} else if fieldDir[1] == "asc" {
gcs.order = append(gcs.order, asc)
} else if fieldDir[1] == "desc" {
gcs.order = append(gcs.order, desc)
} else {
return nil, errors.Errorf("unknown sort direction '%s'", fieldDir[1])
}
}
return gcs, nil
}
// findGroupCounts gets a safe-to-use but possibly empty []GroupCount from
// an interface which might be a *GroupCounts or a []GroupCount.
func findGroupCounts(v interface{}) []GroupCount {
switch gc := v.(type) {
case []GroupCount:
return gc
case *GroupCounts:
return gc.Groups()
}
return nil
}
func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*GroupCounts, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupBy")
defer span.Finish()
// validate call
@ -2594,6 +2735,25 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c
return nil, err
}
var sorter *groupCountSorter
if sortSpec, found, err := c.StringArg("sort"); err != nil {
return nil, errors.Wrap(err, "getting sort arg")
} else if found {
sorter, err = getSorter(sortSpec)
if err != nil {
return nil, errors.Wrap(err, "parsing sort spec")
}
// don't want to prematurely limit the results if we're sorting
limit = int(^uint(0) >> 1)
}
having, hasHaving, err := c.CallArg("having")
if err != nil {
return nil, errors.Wrap(err, "getting 'having' argument")
} else if hasHaving {
// don't want to prematurely limit the results if we're filtering some out
limit = int(^uint(0) >> 1)
}
idx := e.Holder.Index(index)
if idx == nil {
return nil, newNotFoundError(ErrIndexNotFound, index)
@ -2642,7 +2802,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c
return nil, errors.Wrap(err, "getting rows for ")
}
if len(childRows[i]) == 0 { // there are no results because this field has no values.
return []GroupCount{}, nil
return &GroupCounts{}, nil
}
}
}
@ -2653,23 +2813,83 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c
}
// Merge returned results at coordinating node.
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.([]GroupCount)
other := findGroupCounts(prev)
if err := ctx.Err(); err != nil {
return err
}
return mergeGroupCounts(other, v.([]GroupCount), limit)
return mergeGroupCounts(other, findGroupCounts(v), limit)
}
// Get full result set.
other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
return nil, errors.Wrapf(err, "mapReduce shards: %v", shards)
}
results, _ := other.([]GroupCount)
// If there's no sorting, we want to apply limits before
// calculating the Distinct aggregate which is expensive on a
// per-result basis.
if sorter == nil && !hasHaving {
results, err = applyLimitAndOffsetToGroupByResult(c, results)
if err != nil {
return nil, errors.Wrap(err, "applying limit/offset")
}
}
// TODO as an optimization, we could apply some "having"
// conditions here long as they aren't on the Count(Distinct)
// aggregate
// Calculate Count(Distinct) aggregate if requested.
aggregate, _, err := c.CallArg("aggregate")
if err == nil && aggregate != nil && aggregate.Name == "Count" && len(aggregate.Children) > 0 && aggregate.Children[0].Name == "Distinct" && !opt.Remote {
for n, gc := range results {
intersectRows := make([]*pql.Call, 0, len(gc.Group))
for _, fr := range gc.Group {
var value interface{} = fr.RowID
// use fr.Value instead of fr.RowID if set (from int fields)
if fr.Value != nil {
value = &pql.Condition{Op: pql.EQ, Value: *fr.Value}
}
intersectRows = append(intersectRows, &pql.Call{Name: "Row", Args: map[string]interface{}{fr.Field: value}})
}
// apply any filter, if present
if filter != nil {
intersectRows = append(intersectRows, filter)
}
// also intersect with any children of Distinct
if len(aggregate.Children[0].Children) > 0 {
intersectRows = append(intersectRows, aggregate.Children[0].Children[0])
}
countDistinctIntersect := &pql.Call{
Name: "Count",
Children: []*pql.Call{
{
Name: "Distinct",
Children: []*pql.Call{
{
Name: "Intersect",
Children: intersectRows,
},
},
Args: aggregate.Children[0].Args,
Type: pql.PrecallGlobal,
},
},
}
opt.PreTranslated = true
aggregateCount, err := e.execute(ctx, qcx, index, &pql.Query{Calls: []*pql.Call{countDistinctIntersect}}, []uint64{}, opt)
if err != nil {
return nil, err
}
results[n].Agg = int64(aggregateCount[0].(uint64))
}
}
// Apply having.
if having, hasHaving, err := c.CallArg("having"); err != nil {
return nil, err
} else if hasHaving {
if hasHaving && !opt.Remote {
// parse the condition as PQL
if having.Name != "Condition" {
return nil, errors.New("the only supported having call is Condition()")
@ -2687,6 +2907,34 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c
}
}
if sorter != nil && !opt.Remote {
sorter.data = results
sort.Stable(sorter)
results, err = applyLimitAndOffsetToGroupByResult(c, results)
if err != nil {
return nil, errors.Wrap(err, "applying limit/offset")
}
} else if hasHaving && !opt.Remote {
results, err = applyLimitAndOffsetToGroupByResult(c, results)
if err != nil {
return nil, errors.Wrap(err, "applying limit/offset")
}
}
aggType := ""
if aggregate != nil {
switch aggregate.Name {
case "Sum":
aggType = "sum"
case "Count":
aggType = "aggregate"
}
}
return NewGroupCounts(aggType, results...), nil
}
func applyLimitAndOffsetToGroupByResult(c *pql.Call, results []GroupCount) ([]GroupCount, error) {
// Apply offset.
if offset, hasOffset, err := c.UintArg("offset"); err != nil {
return nil, err
@ -2768,17 +3016,70 @@ func (fr FieldRow) String() string {
return fmt.Sprintf("%s.%d.%s", fr.Field, fr.RowID, fr.RowKey)
}
type aggregateType int
const (
nilAggregate aggregateType = 0
sumAggregate aggregateType = 1
distinctAggregate aggregateType = 2
)
// GroupCounts is a list of GroupCount.
type GroupCounts []GroupCount
type GroupCounts struct {
groups []GroupCount
aggregateType aggregateType
}
// AggregateColumn gives the likely column name to use for aggregates, because
// for historical reasons we used "sum" when it was a sum, but don't want to
// use that when it's something else. This will likely get revisited.
func (g *GroupCounts) AggregateColumn() string {
switch g.aggregateType {
case sumAggregate:
return "sum"
case distinctAggregate:
return "aggregate"
default:
return ""
}
}
// Groups is a convenience method to let us not worry as much about the
// potentially-nil nature of a *GroupCounts.
func (g *GroupCounts) Groups() []GroupCount {
if g == nil {
return nil
}
return g.groups
}
// NewGroupCounts creates a GroupCounts with the given type and slice
// of GroupCount objects. There's intentionally no externally-accessible way
// to change the []GroupCount after creation.
func NewGroupCounts(agg string, groups ...GroupCount) *GroupCounts {
var aggType aggregateType
switch agg {
case "sum":
aggType = sumAggregate
case "aggregate":
aggType = distinctAggregate
case "":
aggType = nilAggregate
default:
panic(fmt.Sprintf("invalid aggregate type %q", agg))
}
return &GroupCounts{aggregateType: aggType, groups: groups}
}
// ToTable implements the ToTabler interface.
func (g GroupCounts) ToTable() (*pb.TableResponse, error) {
return pb.RowsToTable(&g, len(g))
func (g *GroupCounts) ToTable() (*pb.TableResponse, error) {
return pb.RowsToTable(g, len(g.Groups()))
}
// ToRows implements the ToRowser interface.
func (g GroupCounts) ToRows(callback func(*pb.RowResponse) error) error {
for i, gc := range g {
func (g *GroupCounts) ToRows(callback func(*pb.RowResponse) error) error {
agg := g.AggregateColumn()
for i, gc := range g.Groups() {
var ci []*pb.ColumnInfo
if i == 0 {
for _, fieldRow := range gc.Group {
@ -2791,7 +3092,10 @@ func (g GroupCounts) ToRows(callback func(*pb.RowResponse) error) error {
}
}
ci = append(ci, &pb.ColumnInfo{Name: "count", Datatype: "uint64"})
ci = append(ci, &pb.ColumnInfo{Name: "sum", Datatype: "int64"})
if agg != "" {
ci = append(ci, &pb.ColumnInfo{Name: agg, Datatype: "int64"})
}
}
rowResp := &pb.RowResponse{
Headers: ci,
@ -2808,9 +3112,11 @@ func (g GroupCounts) ToRows(callback func(*pb.RowResponse) error) error {
}
}
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: gc.Count}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: gc.Sum}},
)
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: gc.Count}})
if agg != "" {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: gc.Agg}})
}
if err := callback(rowResp); err != nil {
return errors.Wrap(err, "calling callback")
}
@ -2818,18 +3124,51 @@ func (g GroupCounts) ToRows(callback func(*pb.RowResponse) error) error {
return nil
}
// MarshalJSON makes GroupCounts satisfy interface json.Marshaler and
// customizes the JSON output of the aggregate field label.
func (g *GroupCounts) MarshalJSON() ([]byte, error) {
groups := g.Groups()
var counts interface{} = groups
if len(groups) == 0 {
return []byte("[]"), nil
}
switch g.aggregateType {
case sumAggregate:
counts = *(*[]groupCountSum)(unsafe.Pointer(&groups))
case distinctAggregate:
counts = *(*[]groupCountAggregate)(unsafe.Pointer(&groups))
}
return json.Marshal(counts)
}
// GroupCount represents a result item for a group by query.
type GroupCount struct {
Group []FieldRow `json:"group"`
Count uint64 `json:"count"`
Sum int64 `json:"sum"`
Agg int64 `json:"-"`
}
type groupCountSum struct {
Group []FieldRow `json:"group"`
Count uint64 `json:"count"`
Agg int64 `json:"sum"`
}
type groupCountAggregate struct {
Group []FieldRow `json:"group"`
Count uint64 `json:"count"`
Agg int64 `json:"aggregate"`
}
var _ GroupCount = GroupCount(groupCountSum{})
var _ GroupCount = GroupCount(groupCountAggregate{})
func (g *GroupCount) Clone() (r *GroupCount) {
r = &GroupCount{
Group: make([]FieldRow, len(g.Group)),
Count: g.Count,
Sum: g.Sum,
Agg: g.Agg,
}
for i := range g.Group {
r.Group[i] = *(g.Group[i].Clone())
@ -2853,7 +3192,7 @@ func mergeGroupCounts(a, b []GroupCount, limit int) []GroupCount {
i++
case 0:
a[i].Count += b[j].Count
a[i].Sum += b[j].Sum
a[i].Agg += b[j].Agg
ret = append(ret, a[i])
i++
j++
@ -2960,27 +3299,27 @@ func (g GroupCount) satisfiesCondition(subj string, cond *pql.Condition) bool {
return false
}
if cond.Op == pql.EQ {
if g.Sum == val {
if g.Agg == val {
return true
}
} else if cond.Op == pql.NEQ {
if g.Sum != val {
if g.Agg != val {
return true
}
} else if cond.Op == pql.LT {
if g.Sum < val {
if g.Agg < val {
return true
}
} else if cond.Op == pql.LTE {
if g.Sum <= val {
if g.Agg <= val {
return true
}
} else if cond.Op == pql.GT {
if g.Sum > val {
if g.Agg > val {
return true
}
} else if cond.Op == pql.GTE {
if g.Sum >= val {
if g.Agg >= val {
return true
}
}
@ -2990,19 +3329,19 @@ func (g GroupCount) satisfiesCondition(subj string, cond *pql.Condition) bool {
return false
}
if cond.Op == pql.BETWEEN {
if val[0] <= g.Sum && g.Sum <= val[1] {
if val[0] <= g.Agg && g.Agg <= val[1] {
return true
}
} else if cond.Op == pql.BTWN_LT_LTE {
if val[0] < g.Sum && g.Sum <= val[1] {
if val[0] < g.Agg && g.Agg <= val[1] {
return true
}
} else if cond.Op == pql.BTWN_LTE_LT {
if val[0] <= g.Sum && g.Sum < val[1] {
if val[0] <= g.Agg && g.Agg < val[1] {
return true
}
} else if cond.Op == pql.BTWN_LT_LT {
if val[0] < g.Sum && g.Sum < val[1] {
if val[0] < g.Agg && g.Agg < val[1] {
return true
}
}
@ -5102,7 +5441,7 @@ loop:
continue loop
}
}
return nil, errShardUnavailable
return nil, errors.Wrapf(errShardUnavailable, "%s:%d:%v:%v", index, shard, shards, nodes)
}
return m, nil
}
@ -5225,7 +5564,7 @@ func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch cha
// Group shards together by nodes.
m, err := e.shardsByNode(nodes, index, shards)
if err != nil {
return errors.Wrap(err, "shards by node")
return errors.Wrapf(err, "shards by node %v", shards)
}
// Execute each node in a separate goroutine.
@ -6273,10 +6612,11 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
}
}
case []GroupCount:
case *GroupCounts:
fieldIDs := make(map[*Field]map[uint64]struct{})
foreignIDs := make(map[*Field]map[uint64]struct{})
for _, gl := range result {
groups := result.Groups()
for _, gl := range groups {
for _, g := range gl.Group {
field := idx.Field(g.Field)
if field == nil {
@ -6287,7 +6627,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
if fi := field.ForeignIndex(); fi != "" {
m, ok := foreignIDs[field]
if !ok {
m = make(map[uint64]struct{}, len(result))
m = make(map[uint64]struct{}, len(groups))
foreignIDs[field] = m
}
@ -6298,7 +6638,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
m, ok := fieldIDs[field]
if !ok {
m = make(map[uint64]struct{}, len(result))
m = make(map[uint64]struct{}, len(groups))
fieldIDs[field] = m
}
@ -6325,8 +6665,11 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
foreignTranslations[field.Name()] = trans
}
other := make([]GroupCount, 0)
for _, gl := range result {
// We are reluctant to smash result, and I'm not sure we need
// to be but I'm not sure we don't need to be.
newGroups := make([]GroupCount, len(groups))
copy(newGroups, groups)
for gi, gl := range groups {
group := make([]FieldRow, len(gl.Group))
for i, g := range gl.Group {
@ -6339,15 +6682,15 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
group[i] = g
}
other = append(other, GroupCount{
Group: group,
Count: gl.Count,
Sum: gl.Sum,
})
// Replace with translated group.
newGroups[gi].Group = group
}
other := &GroupCounts{}
if result != nil {
other.aggregateType = result.aggregateType
}
other.groups = newGroups
return other, nil
case RowIDs:
fieldName := callArgString(call, "_field")
if fieldName == "" {
@ -6628,6 +6971,7 @@ type execOptions struct {
ExcludeRowAttrs bool
ExcludeColumns bool
ColumnAttrs bool
PreTranslated bool
EmbeddedData []*Row
}
@ -7193,7 +7537,7 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool
if gbi.done {
return ret, true, nil
}
if gbi.aggregate == nil {
if gbi.aggregate == nil || gbi.aggregate.Name == "Count" {
if len(gbi.rows) == 1 {
ret.Count = gbi.rows[len(gbi.rows)-1].row.Count()
} else {
@ -7213,7 +7557,7 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool
return ret, false, err
}
ret.Count = uint64(result.Count)
ret.Sum = result.Val
ret.Agg = result.Val
}
}
if ret.Count == 0 {

95
executor_internal_test.go
View file

@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"fmt"
"reflect"
"strconv"
"testing"
@ -163,7 +164,7 @@ func TestExecutor_GroupCountCondition(t *testing.T) {
},
},
{
groupCount: GroupCount{Sum: 100},
groupCount: GroupCount{Agg: 100},
checks: []condCheck{
{cond: "sum == 99", exp: false},
{cond: "sum != 99", exp: true},
@ -195,7 +196,7 @@ func TestExecutor_GroupCountCondition(t *testing.T) {
},
},
{
groupCount: GroupCount{Sum: -100},
groupCount: GroupCount{Agg: -100},
checks: []condCheck{
{cond: "sum == -99", exp: false},
{cond: "sum != -99", exp: true},
@ -384,3 +385,93 @@ func TestToInt64(t *testing.T) {
}
}
}
func TestGetSorter(t *testing.T) {
tests := []struct {
sortSpec string
expGCS *groupCountSorter
expErr string
}{
{
sortSpec: "count asc",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: "count asc",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: " count asc",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: " count asc ",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}},
},
{
sortSpec: "count",
expGCS: &groupCountSorter{fields: []int{-1}, order: []order{desc}},
},
{
sortSpec: "sum asc",
expGCS: &groupCountSorter{fields: []int{-2}, order: []order{asc}},
},
{
sortSpec: "aggregate asc",
expGCS: &groupCountSorter{fields: []int{-2}, order: []order{asc}},
},
{
sortSpec: "boondoggle asc",
expErr: "sorting is only supported on count, aggregate, or sum, not 'boondoggle'",
},
{
sortSpec: "sum asc, count desc",
expGCS: &groupCountSorter{fields: []int{-2, -1}, order: []order{asc, desc}},
},
{
sortSpec: "count asc, sum desc",
expGCS: &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}},
},
{
sortSpec: " count asc , sum desc ",
expGCS: &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}},
},
{
sortSpec: " count asc , sum desc blah",
expErr: "parsing sort directive: 'sum desc blah': too many elements",
},
{
sortSpec: "count asc, sum fesc",
expErr: "unknown sort direction 'fesc'",
},
{
sortSpec: " , sum fesc",
expErr: "invalid sorting directive: ''",
},
{
// weird and useless, but I guess fine?
sortSpec: "count asc,count asc ",
expGCS: &groupCountSorter{fields: []int{-1, -1}, order: []order{asc, asc}},
},
}
for i, tst := range tests {
t.Run(fmt.Sprintf("%s_%d", tst.sortSpec, i), func(t *testing.T) {
gcs, err := getSorter(tst.sortSpec)
if err != nil {
if tst.expErr == "" {
t.Errorf("unexpected error: %v", err)
return
}
if tst.expErr != err.Error() {
t.Errorf("mismatched errors got: '%v', exp: '%s'", err, tst.expErr)
}
return
}
if !reflect.DeepEqual(gcs, tst.expGCS) {
t.Errorf("exp:\n%+v\ngot:\n%v\n", tst.expGCS, gcs)
}
})
}
}

370
executor_test.go
View file

@ -1343,7 +1343,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
t.Fatal(err)
} else if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 100)); err != nil {
t.Fatal(err)
} else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN() on integer field: "f"` {
} else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || !strings.Contains(err.Error(), `finding top results: cannot compute TopN() on integer field: "f"`) {
t.Fatalf("unexpected error: %v", err)
}
})
@ -1362,7 +1362,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
Set(0, f=1)
`}); err != nil {
t.Fatal(err)
} else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN(), field has no cache: "f"` {
} else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || !strings.Contains(err.Error(), `finding top results: cannot compute TopN(), field has no cache: "f"`) {
t.Fatalf("unexpected error: %v", err)
}
})
@ -3066,7 +3066,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4},
}
results := res.Results[0].([]pilosa.GroupCount)
results := res.Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
}
})
@ -3108,7 +3108,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "fint", Value: &d}}, Count: 1},
}
results := res.Results[0].([]pilosa.GroupCount)
results := res.Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
}
})
@ -3139,7 +3139,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "hint", Value: &c}}, Count: 1},
}
results := res.Results[0].([]pilosa.GroupCount)
results := res.Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
}
})
@ -5074,20 +5074,20 @@ func TestExecutor_GroupByStrings(t *testing.T) {
{
query: "GroupBy(Rows(generals), aggregate=Sum(field=v))",
expected: []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Sum: 25},
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Sum: 30},
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Agg: 25},
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 30},
},
},
{
query: "GroupBy(Rows(generals), aggregate=Sum(field=v), having=Condition(sum>25))",
expected: []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Sum: 30},
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 30},
},
},
{
query: "GroupBy(Rows(generals), aggregate=Sum(field=v), having=Condition(-5<sum<27))",
expected: []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Sum: 25},
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Agg: 25},
},
},
{
@ -5100,52 +5100,52 @@ func TestExecutor_GroupByStrings(t *testing.T) {
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "v", Value: &v1}},
Count: 1,
Sum: 0,
Agg: 0,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "v", Value: &v2}},
Count: 1,
Sum: 0,
Agg: 0,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "v", Value: &v3}},
Count: 1,
Sum: 0,
Agg: 0,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "v", Value: &v4}},
Count: 1,
Sum: 0,
Agg: 0,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "v", Value: &v5}},
Count: 1,
Sum: 0,
Agg: 0,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "v", Value: &v6}},
Count: 1,
Sum: 0,
Agg: 0,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "v", Value: &v7}},
Count: 1,
Sum: 0,
Agg: 0,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "v", Value: &v8}},
Count: 1,
Sum: 0,
Agg: 0,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "v", Value: &v9}},
Count: 1,
Sum: 0,
Agg: 0,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "v", Value: &v10}},
Count: 1,
Sum: 0,
Agg: 0,
},
},
},
@ -5155,12 +5155,12 @@ func TestExecutor_GroupByStrings(t *testing.T) {
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "vv", Value: &v3}},
Count: 3,
Sum: 9,
Agg: 9,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "vv", Value: &v4}},
Count: 4,
Sum: 16,
Agg: 16,
},
},
},
@ -5170,12 +5170,12 @@ func TestExecutor_GroupByStrings(t *testing.T) {
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "nv", Value: &nv4}},
Count: 4,
Sum: -16,
Agg: -16,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "nv", Value: &nv3}},
Count: 3,
Sum: -9,
Agg: -9,
},
},
},
@ -5185,12 +5185,12 @@ func TestExecutor_GroupByStrings(t *testing.T) {
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "nv", Value: &nv4}},
Count: 4,
Sum: -16,
Agg: -16,
},
{
Group: []pilosa.FieldRow{pilosa.FieldRow{Field: "nv", Value: &nv3}},
Count: 3,
Sum: -9,
Agg: -9,
},
},
},
@ -5203,7 +5203,7 @@ func TestExecutor_GroupByStrings(t *testing.T) {
pilosa.FieldRow{Field: "nv", Value: &nv3},
},
Count: 3,
Sum: 9,
Agg: 9,
},
{
Group: []pilosa.FieldRow{
@ -5211,7 +5211,7 @@ func TestExecutor_GroupByStrings(t *testing.T) {
pilosa.FieldRow{Field: "nv", Value: &nv4},
},
Count: 4,
Sum: 16,
Agg: 16,
},
},
},
@ -5226,7 +5226,7 @@ func TestExecutor_GroupByStrings(t *testing.T) {
if err != nil {
t.Fatal(err)
}
results := r.Results[0].([]pilosa.GroupCount)
results := r.Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, tst.expected, results)
})
}
@ -5545,6 +5545,8 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
t.Fatal(err)
} else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, v=100)`}); err != nil {
t.Fatal(err)
} else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, v=100)`, ShardWidth+10)}); err != nil { // Workaround distinct bug where v must be set in every shard
t.Fatal(err)
}
t.Run("No Field List Arguments", func(t *testing.T) {
@ -5573,7 +5575,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1},
}
results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(sub))`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(sub))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
@ -5585,7 +5587,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1},
}
results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub))`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
@ -5595,17 +5597,50 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1},
}
results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), filter=Row(general=10))`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), filter=Row(general=10))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
t.Run("Aggregate", func(t *testing.T) {
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 2, Sum: 110},
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1, Sum: 10},
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 2, Agg: 110},
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 10},
}
results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), aggregate=Sum(field=v))`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), aggregate=Sum(field=v))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
t.Run("AggregateCountDistinct", func(t *testing.T) {
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3, Agg: 2},
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 1},
{Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 0},
{Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 0},
}
results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), aggregate=Count(Distinct(field=v)))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
t.Run("AggregateCountDistinctFilter", func(t *testing.T) {
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 1, Agg: 1},
}
results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), filter=Row(v > 10), aggregate=Count(Distinct(field=v)))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
t.Run("AggregateCountDistinctFilterDistinct", func(t *testing.T) {
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3, Agg: 1},
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 0},
{Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 0},
{Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 0},
}
results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), aggregate=Count(Distinct(Row(v > 10), field=v)))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
@ -5615,7 +5650,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2},
}
results := c.Query(t, "i", `GroupBy(Rows(general, previous=10))`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(general, previous=10))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
@ -5624,7 +5659,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2},
}
results := c.Query(t, "i", `GroupBy(Rows(general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(general, previous=10), limit=1)`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
@ -5645,7 +5680,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "a", RowID: 0}, {Field: "b", RowID: 1}}, Count: 1},
}
results := c.Query(t, "i", `GroupBy(Rows(a), Rows(b), limit=1)`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(a), Rows(b), limit=1)`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
@ -5673,7 +5708,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
})
t.Run("test wrapping with previous", func(t *testing.T) {
results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb), Rows(wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb), Rows(wc, previous=1), limit=3)`).Results[0].(*pilosa.GroupCounts).Groups()
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 2}}, Count: 2},
{Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1},
@ -5683,14 +5718,14 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
})
t.Run("test previous is last result", func(t *testing.T) {
results := c.Query(t, "i", `GroupBy(Rows(wa, previous=3), Rows(wb, previous=3), Rows(wc, previous=3), limit=3)`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(wa, previous=3), Rows(wb, previous=3), Rows(wc, previous=3), limit=3)`).Results[0].(*pilosa.GroupCounts).Groups()
if len(results) > 0 {
t.Fatalf("expected no results because previous specified last result")
}
})
t.Run("test wrapping multiple", func(t *testing.T) {
results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb, previous=2), Rows(wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb, previous=2), Rows(wc, previous=2), limit=1)`).Results[0].(*pilosa.GroupCounts).Groups()
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1},
}
@ -5714,7 +5749,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{3, ShardWidth},
})
t.Run("distinct rows in different shards", func(t *testing.T) {
results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb), limit=5)`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb), limit=5)`).Results[0].(*pilosa.GroupCounts).Groups()
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 2}}, Count: 1},
@ -5726,7 +5761,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
})
t.Run("distinct rows in different shards with row limit", func(t *testing.T) {
results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb, limit=2), limit=5)`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb, limit=2), limit=5)`).Results[0].(*pilosa.GroupCounts).Groups()
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1},
@ -5737,7 +5772,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
})
t.Run("distinct rows in different shards with column arg", func(t *testing.T) {
results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(ma), Rows(mb, column=%d), limit=5)`, ShardWidth)).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(ma), Rows(mb, column=%d), limit=5)`, ShardWidth)).Results[0].(*pilosa.GroupCounts).Groups()
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1},
@ -5762,7 +5797,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{1, ShardWidth},
})
t.Run("same rows in different shards", func(t *testing.T) {
results := c.Query(t, "i", `GroupBy(Rows(na), Rows(nb))`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(na), Rows(nb))`).Results[0].(*pilosa.GroupCounts).Groups()
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 0}}, Count: 2},
{Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 1}}, Count: 2},
@ -5799,12 +5834,12 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
t.Run("test wrapping with previous", func(t *testing.T) {
totalResults := make([]pilosa.GroupCount, 0)
results := c.Query(t, "i", `GroupBy(Rows(ppa), Rows(ppb), Rows(ppc), limit=3)`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(ppa), Rows(ppb), Rows(ppc), limit=3)`).Results[0].(*pilosa.GroupCounts).Groups()
totalResults = append(totalResults, results...)
for len(totalResults) < 64 {
lastGroup := results[len(results)-1].Group
query := fmt.Sprintf("GroupBy(Rows(ppa, previous=%d), Rows(ppb, previous=%d), Rows(ppc, previous=%d), limit=3)", lastGroup[0].RowID, lastGroup[1].RowID, lastGroup[2].RowID)
results = c.Query(t, "i", query).Results[0].([]pilosa.GroupCount)
results = c.Query(t, "i", query).Results[0].(*pilosa.GroupCounts).Groups()
totalResults = append(totalResults, results...)
}
@ -5846,7 +5881,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "generalk", RowID: 3, RowKey: "twelve"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1},
}
results := c.Query(t, "i", `GroupBy(Rows(generalk), Rows(subk))`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "i", `GroupBy(Rows(generalk), Rows(subk))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
@ -5884,7 +5919,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "child", RowID: 2, RowKey: "three"}}, Count: 2},
}
results := c.Query(t, "fic", `GroupBy(Rows(child))`).Results[0].([]pilosa.GroupCount)
results := c.Query(t, "fic", `GroupBy(Rows(child))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
})
@ -6470,10 +6505,11 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) {
if err != nil {
t.Fatal(err)
}
gc, ok := resp.Results[0].([]pilosa.GroupCount)
gcc, ok := resp.Results[0].(*pilosa.GroupCounts)
if !ok {
t.Fatalf("invalid response type, expected: []pilosa.GroupCount, got: %T", resp.Results[0])
}
gc := gcc.Groups()
if len(gc) != 2 {
t.Fatalf("invalid group count length, expected: 2, got: %v", len(gc))
}
@ -6780,7 +6816,15 @@ func TestMissingKeyRegression(t *testing.T) {
// (single and multi-node clusters, different endpoints for the
// queries (HTTP, GRPC, Postgres), etc.).
func TestVariousQueries(t *testing.T) {
c := test.MustRunCluster(t, 3)
for _, clusterSize := range []int{1, 3, 4, 7} {
t.Run(fmt.Sprintf("%d-node", clusterSize), func(t *testing.T) {
variousQueries(t, clusterSize)
})
}
}
func variousQueries(t *testing.T, clusterSize int) {
c := test.MustRunCluster(t, clusterSize)
defer c.Close()
// Create and populate "likenums" similar to "likes", but without keys on the field.
@ -6796,7 +6840,9 @@ func TestVariousQueries(t *testing.T) {
{ID: 7, Key: "userB"},
{ID: 7, Key: "userC"},
{ID: 7, Key: "userD"},
{ID: 7, Key: "userE"},
// we intentionally leave user E out because then there is no
// data for userE's shard for this field, which triggered a
// "fragment not found" problem
{ID: 7, Key: "userF"},
})
@ -6827,6 +6873,28 @@ func TestVariousQueries(t *testing.T) {
{Val: 0, Key: "userE"},
})
// Create and populate "affinity" int field with negative, positive, zero and null values.
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "net_worth", pilosa.OptFieldTypeInt(-100000000, 100000000))
c.ImportIntKey(t, "users", "net_worth", []test.IntKey{
{Val: 1, Key: "userA"},
{Val: 10, Key: "userB"},
{Val: 100, Key: "userC"},
{Val: 1000, Key: "userD"},
{Val: 10000, Key: "userE"},
{Val: 100000, Key: "userF"},
})
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "zip_code", pilosa.OptFieldTypeInt(0, 100000))
c.ImportIntKey(t, "users", "zip_code", []test.IntKey{
{Val: 78739, Key: "userA"},
{Val: 78739, Key: "userB"},
{Val: 19707, Key: "userC"},
{Val: 19707, Key: "userD"},
{Val: 86753, Key: "userE"},
{Val: 78739, Key: "userG"},
})
tests := []struct {
query string
qrVerifier func(t *testing.T, resp pilosa.QueryResponse)
@ -6835,11 +6903,11 @@ func TestVariousQueries(t *testing.T) {
{
query: "Count(All())",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if resp.Results[0].(uint64) != 6 {
t.Errorf("expected 6, got %+v", resp.Results[0])
if resp.Results[0].(uint64) != 7 {
t.Errorf("expected 7, got %+v", resp.Results[0])
}
},
csvVerifier: "6\n",
csvVerifier: "7\n",
},
{
query: "Count(Distinct(field=likenums))",
@ -6970,6 +7038,116 @@ func TestVariousQueries(t *testing.T) {
},
csvVerifier: "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n",
},
{
query: "GroupBy(Rows(field=likes))",
csvVerifier: `molecula,1
pilosa,1
pangolin,1
zebra,1
toucan,1
dog,1
icecream,6
`,
},
{
query: "GroupBy(Rows(field=likes), aggregate=Sum(field=net_worth), limit=2, having=Condition(sum>10))",
csvVerifier: `pangolin,1,100
zebra,1,1000
`,
},
{
query: "GroupBy(Rows(field=likes), having=Condition(count>5))",
csvVerifier: "icecream,6\n",
},
{
query: "GroupBy(Rows(field=likes), filter=Row(affinity>-7))",
csvVerifier: `molecula,1
pangolin,1
zebra,1
toucan,1
icecream,4
`,
},
{
query: "GroupBy(Rows(field=likes), aggregate=Count(Distinct(field=zip_code)))",
csvVerifier: `molecula,1,1
pilosa,1,1
pangolin,1,1
zebra,1,1
toucan,1,1
dog,1,0
icecream,6,3
`,
},
{
query: "GroupBy(Rows(field=likes), aggregate=Count(Distinct(field=zip_code)), having=Condition(sum>2))",
csvVerifier: "icecream,6,3\n",
},
{
query: "GroupBy(Rows(field=likes), filter=Row(affinity>-11), aggregate=Count(Distinct(field=zip_code)))",
csvVerifier: `molecula,1,1
pilosa,1,1
pangolin,1,1
zebra,1,1
toucan,1,1
icecream,5,3
`,
},
{
query: "GroupBy(Rows(field=likes), filter=Row(affinity>-11), aggregate=Count(Distinct(Row(affinity>-7), field=zip_code)))",
csvVerifier: `molecula,1,1
pilosa,1,0
pangolin,1,1
zebra,1,1
toucan,1,1
icecream,5,3
`,
},
{
query: "GroupBy(Rows(field=likes), sort=\"count desc\")",
csvVerifier: `icecream,6
molecula,1
pilosa,1
pangolin,1
zebra,1
toucan,1
dog,1
`,
},
{
query: "GroupBy(Rows(field=likes), aggregate=Sum(field=net_worth), sort=\"aggregate desc, count asc\")",
csvVerifier: `icecream,6,111111
dog,1,100000
toucan,1,10000
zebra,1,1000
pangolin,1,100
pilosa,1,10
molecula,1,1
`,
},
{
query: "GroupBy(Rows(field=likes), aggregate=Sum(field=net_worth), sort=\"aggregate desc, count asc\", limit=3)",
csvVerifier: `icecream,6,111111
dog,1,100000
toucan,1,10000
`,
},
{
query: "GroupBy(Rows(field=likes), aggregate=Sum(field=net_worth),sort=\"aggregate desc, count asc\",limit=3,offset=2)",
csvVerifier: `toucan,1,10000
zebra,1,1000
pangolin,1,100
`,
},
{
query: "GroupBy(Rows(field=affinity), aggregate=Count(Distinct(field=zip_code)))",
csvVerifier: `-10,1,1
-5,1,1
0,1,1
5,1,1
10,1,1
`,
},
}
for i, tst := range tests {
@ -6986,7 +7164,7 @@ func TestVariousQueries(t *testing.T) {
// verify everything after header
got := csvString[strings.Index(csvString, "\n")+1:]
if got != tst.csvVerifier {
t.Errorf("expected '%s', got '%s'", tst.csvVerifier, got)
t.Errorf("expected:\n%s\ngot:\n%s", tst.csvVerifier, got)
}
// TODO: add HTTP and Postgres and ability to convert
@ -6995,6 +7173,88 @@ func TestVariousQueries(t *testing.T) {
}
}
// TestVariousSingleShardQueries tests queries on a dataset which
// consists of an unkeyed index, and data only in the first
// shard. Turns out that there are some interesting failure modes
// which only crop up with one shard. An example is that the mapReduce
// logic does its first reduce call with a nil interface{} value, and
// an actual result value. If this is the only reduce that gets done
// (because there's only one shard), the result might never go through
// normal merge/reduction logic and might e.g. be nil instead of an
// empty struct resulting in an NPE later on.
func TestVariousSingleShardQueries(t *testing.T) {
for _, clusterSize := range []int{1, 4} {
t.Run(fmt.Sprintf("%d-node", clusterSize), func(t *testing.T) {
variousSingleShardQueries(t, clusterSize)
})
}
}
func variousSingleShardQueries(t *testing.T, clusterSize int) {
c := test.MustRunCluster(t, clusterSize)
defer c.Close()
// Create and populate "likenums" similar to "likes", but without keys on the field.
c.CreateField(t, "events", pilosa.IndexOptions{Keys: false, TrackExistence: true}, "lostcount", pilosa.OptFieldTypeInt(0, 1000000000))
c.ImportIntID(t, "events", "lostcount", []test.IntID{
{Val: 0, ID: 1},
{Val: 1, ID: 2},
{Val: 0, ID: 3},
{Val: 2, ID: 4},
{Val: 2, ID: 5},
{Val: 0, ID: 6},
{Val: 3, ID: 7},
{Val: 3, ID: 8},
{Val: 3, ID: 9},
{Val: 0, ID: 10},
})
c.CreateField(t, "events", pilosa.IndexOptions{Keys: false, TrackExistence: true}, "jittermax", pilosa.OptFieldTypeInt(0, 1000000000))
c.ImportIntID(t, "events", "jittermax", []test.IntID{
{Val: 17, ID: 1},
{Val: 3, ID: 2},
{Val: 42, ID: 3},
{Val: 9, ID: 4},
{Val: 17, ID: 5},
{Val: 3, ID: 6},
{Val: 42, ID: 7},
{Val: 9, ID: 8},
{Val: 17, ID: 9},
{Val: 3, ID: 10},
})
tests := []struct {
query string
csvVerifier string
}{
{
query: "GroupBy(Rows(lostcount), aggregate=Count(Distinct(field=jittermax)))",
csvVerifier: `0,4,3
1,1,1
2,2,2
3,3,3
`,
},
}
for i, tst := range tests {
t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) {
tr := c.QueryGRPC(t, "events", tst.query)
csvString, err := tableResponseToCSVString(tr)
if err != nil {
t.Fatal(err)
}
// verify everything after header
got := csvString[strings.Index(csvString, "\n")+1:]
if got != tst.csvVerifier {
t.Errorf("expected:\n%s\ngot:\n%s", tst.csvVerifier, got)
}
})
}
}
// tableResponseToCSV converts a generic TableResponse to a CSV format
// and writes it to the writer.
func tableResponseToCSV(m *proto.TableResponse, w io.Writer) error {

4
handler.go
View file

@ -46,6 +46,10 @@ type QueryRequest struct {
// If false, this request is on the originating node.
Remote bool
// Query has already been translated. This is only used if Remote
// is false, Remote=true implies this.
PreTranslated bool
// Should we profile this query?
Profile bool

8
http/client.go
View file

@ -286,7 +286,6 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
} else if queryRequest.Query == "" {
return nil, pilosa.ErrQueryRequired
}
buf, err := c.serializer.Marshal(queryRequest)
if err != nil {
return nil, errors.Wrap(err, "marshaling queryRequest")
@ -308,7 +307,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
// Execute request against the host.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
return nil, err
return nil, errors.Wrapf(err, "'%s', shards %v", queryRequest.Query, queryRequest.Shards)
}
defer resp.Body.Close()
@ -1714,12 +1713,15 @@ func (c *InternalClient) executeRequest(req *http.Request, opts ...executeReques
var msg string
// try to decode a JSON response
var sr successResponse
qr := &pilosa.QueryResponse{}
if err = json.Unmarshal(buf, &sr); err == nil {
msg = sr.Error.Error()
} else if err := c.serializer.Unmarshal(buf, qr); err == nil {
msg = qr.Err.Error()
} else {
msg = string(buf)
}
return resp, errors.Errorf("server error %s: '%s'", resp.Status, msg)
return resp, errors.Errorf("against %s %s: '%s'", req.URL.String(), resp.Status, msg)
}
return resp, nil
}

192
internal/private.pb.go
View file

@ -5620,10 +5620,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -6028,10 +6025,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -6114,10 +6108,7 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -6302,10 +6293,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -6508,10 +6496,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -6638,10 +6623,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -6788,7 +6770,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > postIndex {
@ -6805,10 +6787,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -6942,10 +6921,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -7028,10 +7004,7 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -7169,10 +7142,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -7342,10 +7312,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -7460,10 +7427,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -7597,10 +7561,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -7770,10 +7731,7 @@ func (m *Field) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -7858,10 +7816,7 @@ func (m *Schema) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -8033,10 +7988,7 @@ func (m *Index) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -8170,10 +8122,7 @@ func (m *URI) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -8380,10 +8329,7 @@ func (m *Node) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -8498,10 +8444,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -8607,10 +8550,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -8767,10 +8707,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -8906,10 +8843,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -9087,10 +9021,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -9275,10 +9206,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -9431,10 +9359,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -9581,10 +9506,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -9731,10 +9653,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -10016,10 +9935,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -10221,10 +10137,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -10362,10 +10275,7 @@ func (m *TranslationResizeSource) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -10503,10 +10413,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -10593,10 +10500,7 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -10683,10 +10587,7 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -10801,10 +10702,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -10855,10 +10753,7 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -10977,10 +10872,7 @@ func (m *TransactionMessage) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -11177,10 +11069,7 @@ func (m *Transaction) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
@ -11231,10 +11120,7 @@ func (m *TransactionStats) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {

973
internal/public.pb.go
View file

File diff suppressed because it is too large Load diff

19
internal/public.proto
View file

@ -102,7 +102,7 @@ message FieldRow {
message GroupCount{
repeated FieldRow Group = 1;
uint64 Count = 2;
int64 Sum = 3;
int64 Agg = 3;
}
message ValCount {
@ -144,6 +144,7 @@ message QueryRequest {
bool ExcludeRowAttrs = 6;
bool ExcludeColumns = 7;
repeated Row EmbeddedData = 8;
bool PreTranslated = 9;
}
message QueryResponse {
@ -158,9 +159,15 @@ message QueryResult {
uint64 N = 2;
repeated Pair Pairs = 3;
bool Changed = 4;
ValCount ValCount = 5;
ValCount ValCount = 5;
repeated uint64 RowIDs = 7;
repeated GroupCount GroupCounts = 8;
// In the past, GroupCounts was a []GroupCount which did not indicate
// whether it had an aggregate, or which aggregate it had. We've
// updated this, but we keep this here so that messages using the old
// format can get a best-effort treatment rather than causing panics.
// Later this can almost certainly go away, but leave a comment warning
// people that 8 is Spoken For if you do that, please.
repeated GroupCount OldGroupCounts = 8;
RowIdentifiers RowIdentifiers = 9;
SignedRow SignedRow = 10;
PairsField PairsField = 11;
@ -168,6 +175,7 @@ message QueryResult {
ExtractedIDMatrix ExtractedIDMatrix = 13;
ExtractedTable ExtractedTable = 14;
RowMatrix RowMatrix = 15;
GroupCounts GroupCounts = 16;
}
message ImportRequest {
@ -253,3 +261,8 @@ message ImportColumnAttrsRequest {
repeated uint64 ColumnIDs = 5;
int64 IndexCreatedAt = 6;
}
message GroupCounts{
string Aggregate = 1;
repeated GroupCount Groups = 2;
}

2
pql/ast.go
View file

@ -441,9 +441,11 @@ var callInfoByFunc = map[string]callInfo{
prototypes: map[string]interface{}{
"filter": nil,
"limit": int64(0),
"offset": int64(0),
"previous": nil,
"aggregate": nil,
"having": nil,
"sort": "",
},
},
"Options": {

596
proto/pilosa.pb.go
View file

@ -838,6 +838,341 @@ func (*IdsOrKeys) XXX_OneofWrappers() []interface{} {
}
}
type Index struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Index) Reset() { *m = Index{} }
func (m *Index) String() string { return proto.CompactTextString(m) }
func (*Index) ProtoMessage() {}
func (*Index) Descriptor() ([]byte, []int) {
return fileDescriptor_ef0691a44d1e275c, []int{13}
}
func (m *Index) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Index.Unmarshal(m, b)
}
func (m *Index) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Index.Marshal(b, m, deterministic)
}
func (m *Index) XXX_Merge(src proto.Message) {
xxx_messageInfo_Index.Merge(m, src)
}
func (m *Index) XXX_Size() int {
return xxx_messageInfo_Index.Size(m)
}
func (m *Index) XXX_DiscardUnknown() {
xxx_messageInfo_Index.DiscardUnknown(m)
}
var xxx_messageInfo_Index proto.InternalMessageInfo
func (m *Index) GetName() string {
if m != nil {
return m.Name
}
return ""
}
type CreateIndexRequest struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
Keys bool `protobuf:"varint,2,opt,name=keys,proto3" json:"keys,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *CreateIndexRequest) Reset() { *m = CreateIndexRequest{} }
func (m *CreateIndexRequest) String() string { return proto.CompactTextString(m) }
func (*CreateIndexRequest) ProtoMessage() {}
func (*CreateIndexRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_ef0691a44d1e275c, []int{14}
}
func (m *CreateIndexRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_CreateIndexRequest.Unmarshal(m, b)
}
func (m *CreateIndexRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_CreateIndexRequest.Marshal(b, m, deterministic)
}
func (m *CreateIndexRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_CreateIndexRequest.Merge(m, src)
}
func (m *CreateIndexRequest) XXX_Size() int {
return xxx_messageInfo_CreateIndexRequest.Size(m)
}
func (m *CreateIndexRequest) XXX_DiscardUnknown() {
xxx_messageInfo_CreateIndexRequest.DiscardUnknown(m)
}
var xxx_messageInfo_CreateIndexRequest proto.InternalMessageInfo
func (m *CreateIndexRequest) GetName() string {
if m != nil {
return m.Name
}
return ""
}
func (m *CreateIndexRequest) GetKeys() bool {
if m != nil {
return m.Keys
}
return false
}
type CreateIndexResponse struct {
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *CreateIndexResponse) Reset() { *m = CreateIndexResponse{} }
func (m *CreateIndexResponse) String() string { return proto.CompactTextString(m) }
func (*CreateIndexResponse) ProtoMessage() {}
func (*CreateIndexResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_ef0691a44d1e275c, []int{15}
}
func (m *CreateIndexResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_CreateIndexResponse.Unmarshal(m, b)
}
func (m *CreateIndexResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_CreateIndexResponse.Marshal(b, m, deterministic)
}
func (m *CreateIndexResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_CreateIndexResponse.Merge(m, src)
}
func (m *CreateIndexResponse) XXX_Size() int {
return xxx_messageInfo_CreateIndexResponse.Size(m)
}
func (m *CreateIndexResponse) XXX_DiscardUnknown() {
xxx_messageInfo_CreateIndexResponse.DiscardUnknown(m)
}
var xxx_messageInfo_CreateIndexResponse proto.InternalMessageInfo
type GetIndexRequest struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *GetIndexRequest) Reset() { *m = GetIndexRequest{} }
func (m *GetIndexRequest) String() string { return proto.CompactTextString(m) }
func (*GetIndexRequest) ProtoMessage() {}
func (*GetIndexRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_ef0691a44d1e275c, []int{16}
}
func (m *GetIndexRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetIndexRequest.Unmarshal(m, b)
}
func (m *GetIndexRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_GetIndexRequest.Marshal(b, m, deterministic)
}
func (m *GetIndexRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_GetIndexRequest.Merge(m, src)
}
func (m *GetIndexRequest) XXX_Size() int {
return xxx_messageInfo_GetIndexRequest.Size(m)
}
func (m *GetIndexRequest) XXX_DiscardUnknown() {
xxx_messageInfo_GetIndexRequest.DiscardUnknown(m)
}
var xxx_messageInfo_GetIndexRequest proto.InternalMessageInfo
func (m *GetIndexRequest) GetName() string {
if m != nil {
return m.Name
}
return ""
}
type GetIndexResponse struct {
Index *Index `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *GetIndexResponse) Reset() { *m = GetIndexResponse{} }
func (m *GetIndexResponse) String() string { return proto.CompactTextString(m) }
func (*GetIndexResponse) ProtoMessage() {}
func (*GetIndexResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_ef0691a44d1e275c, []int{17}
}
func (m *GetIndexResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetIndexResponse.Unmarshal(m, b)
}
func (m *GetIndexResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_GetIndexResponse.Marshal(b, m, deterministic)
}
func (m *GetIndexResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_GetIndexResponse.Merge(m, src)
}
func (m *GetIndexResponse) XXX_Size() int {
return xxx_messageInfo_GetIndexResponse.Size(m)
}
func (m *GetIndexResponse) XXX_DiscardUnknown() {
xxx_messageInfo_GetIndexResponse.DiscardUnknown(m)
}
var xxx_messageInfo_GetIndexResponse proto.InternalMessageInfo
func (m *GetIndexResponse) GetIndex() *Index {
if m != nil {
return m.Index
}
return nil
}
type GetIndexesRequest struct {
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *GetIndexesRequest) Reset() { *m = GetIndexesRequest{} }
func (m *GetIndexesRequest) String() string { return proto.CompactTextString(m) }
func (*GetIndexesRequest) ProtoMessage() {}
func (*GetIndexesRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_ef0691a44d1e275c, []int{18}
}
func (m *GetIndexesRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetIndexesRequest.Unmarshal(m, b)
}
func (m *GetIndexesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_GetIndexesRequest.Marshal(b, m, deterministic)
}
func (m *GetIndexesRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_GetIndexesRequest.Merge(m, src)
}
func (m *GetIndexesRequest) XXX_Size() int {
return xxx_messageInfo_GetIndexesRequest.Size(m)
}
func (m *GetIndexesRequest) XXX_DiscardUnknown() {
xxx_messageInfo_GetIndexesRequest.DiscardUnknown(m)
}
var xxx_messageInfo_GetIndexesRequest proto.InternalMessageInfo
type GetIndexesResponse struct {
Indexes []*Index `protobuf:"bytes,1,rep,name=indexes,proto3" json:"indexes,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *GetIndexesResponse) Reset() { *m = GetIndexesResponse{} }
func (m *GetIndexesResponse) String() string { return proto.CompactTextString(m) }
func (*GetIndexesResponse) ProtoMessage() {}
func (*GetIndexesResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_ef0691a44d1e275c, []int{19}
}
func (m *GetIndexesResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetIndexesResponse.Unmarshal(m, b)
}
func (m *GetIndexesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_GetIndexesResponse.Marshal(b, m, deterministic)
}
func (m *GetIndexesResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_GetIndexesResponse.Merge(m, src)
}
func (m *GetIndexesResponse) XXX_Size() int {
return xxx_messageInfo_GetIndexesResponse.Size(m)
}
func (m *GetIndexesResponse) XXX_DiscardUnknown() {
xxx_messageInfo_GetIndexesResponse.DiscardUnknown(m)
}
var xxx_messageInfo_GetIndexesResponse proto.InternalMessageInfo
func (m *GetIndexesResponse) GetIndexes() []*Index {
if m != nil {
return m.Indexes
}
return nil
}
type DeleteIndexRequest struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *DeleteIndexRequest) Reset() { *m = DeleteIndexRequest{} }
func (m *DeleteIndexRequest) String() string { return proto.CompactTextString(m) }
func (*DeleteIndexRequest) ProtoMessage() {}
func (*DeleteIndexRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_ef0691a44d1e275c, []int{20}
}
func (m *DeleteIndexRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_DeleteIndexRequest.Unmarshal(m, b)
}
func (m *DeleteIndexRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_DeleteIndexRequest.Marshal(b, m, deterministic)
}
func (m *DeleteIndexRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_DeleteIndexRequest.Merge(m, src)
}
func (m *DeleteIndexRequest) XXX_Size() int {
return xxx_messageInfo_DeleteIndexRequest.Size(m)
}
func (m *DeleteIndexRequest) XXX_DiscardUnknown() {
xxx_messageInfo_DeleteIndexRequest.DiscardUnknown(m)
}
var xxx_messageInfo_DeleteIndexRequest proto.InternalMessageInfo
func (m *DeleteIndexRequest) GetName() string {
if m != nil {
return m.Name
}
return ""
}
type DeleteIndexResponse struct {
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *DeleteIndexResponse) Reset() { *m = DeleteIndexResponse{} }
func (m *DeleteIndexResponse) String() string { return proto.CompactTextString(m) }
func (*DeleteIndexResponse) ProtoMessage() {}
func (*DeleteIndexResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_ef0691a44d1e275c, []int{21}
}
func (m *DeleteIndexResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_DeleteIndexResponse.Unmarshal(m, b)
}
func (m *DeleteIndexResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_DeleteIndexResponse.Marshal(b, m, deterministic)
}
func (m *DeleteIndexResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_DeleteIndexResponse.Merge(m, src)
}
func (m *DeleteIndexResponse) XXX_Size() int {
return xxx_messageInfo_DeleteIndexResponse.Size(m)
}
func (m *DeleteIndexResponse) XXX_DiscardUnknown() {
xxx_messageInfo_DeleteIndexResponse.DiscardUnknown(m)
}
var xxx_messageInfo_DeleteIndexResponse proto.InternalMessageInfo
func init() {
proto.RegisterType((*QueryPQLRequest)(nil), "pilosa.QueryPQLRequest")
proto.RegisterType((*QuerySQLRequest)(nil), "pilosa.QuerySQLRequest")
@ -852,60 +1187,79 @@ func init() {
proto.RegisterType((*Uint64Array)(nil), "pilosa.Uint64Array")
proto.RegisterType((*StringArray)(nil), "pilosa.StringArray")
proto.RegisterType((*IdsOrKeys)(nil), "pilosa.IdsOrKeys")
proto.RegisterType((*Index)(nil), "pilosa.Index")
proto.RegisterType((*CreateIndexRequest)(nil), "pilosa.CreateIndexRequest")
proto.RegisterType((*CreateIndexResponse)(nil), "pilosa.CreateIndexResponse")
proto.RegisterType((*GetIndexRequest)(nil), "pilosa.GetIndexRequest")
proto.RegisterType((*GetIndexResponse)(nil), "pilosa.GetIndexResponse")
proto.RegisterType((*GetIndexesRequest)(nil), "pilosa.GetIndexesRequest")
proto.RegisterType((*GetIndexesResponse)(nil), "pilosa.GetIndexesResponse")
proto.RegisterType((*DeleteIndexRequest)(nil), "pilosa.DeleteIndexRequest")
proto.RegisterType((*DeleteIndexResponse)(nil), "pilosa.DeleteIndexResponse")
}
func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) }
var fileDescriptor_ef0691a44d1e275c = []byte{
// 761 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xcd, 0x72, 0xd3, 0x48,
0x10, 0xb6, 0x22, 0xc5, 0xb6, 0xda, 0xf9, 0xdb, 0xc9, 0x6e, 0x56, 0x95, 0xda, 0xda, 0x55, 0x94,
0xc3, 0x7a, 0x6b, 0xb7, 0x92, 0xac, 0x77, 0x03, 0x05, 0x84, 0x43, 0x12, 0xa0, 0x9c, 0x02, 0x0a,
0x67, 0x42, 0x72, 0xe0, 0x36, 0xb6, 0xc6, 0x8e, 0x8a, 0xb1, 0xc6, 0xd6, 0x48, 0x09, 0x7e, 0x01,
0xde, 0x87, 0x33, 0x17, 0x4e, 0x3c, 0x17, 0x35, 0x33, 0x1a, 0xd9, 0x0a, 0x98, 0x0a, 0x54, 0x71,
0xf2, 0x74, 0x7f, 0x5f, 0xb7, 0xfa, 0x9b, 0xee, 0x69, 0xc3, 0xd2, 0x28, 0x62, 0x5c, 0x90, 0x9d,
0x51, 0xc2, 0x53, 0x8e, 0xaa, 0xda, 0x0a, 0xee, 0xc1, 0xea, 0x69, 0x46, 0x93, 0x49, 0xe7, 0xf4,
0x19, 0xa6, 0xe3, 0x8c, 0x8a, 0x14, 0xfd, 0x0c, 0x8b, 0x51, 0x1c, 0xd2, 0x37, 0x9e, 0xe5, 0x5b,
0x4d, 0x17, 0x6b, 0x03, 0xad, 0x81, 0x3d, 0x1a, 0x33, 0x6f, 0x41, 0xf9, 0xe4, 0x31, 0xd8, 0xce,
0x43, 0xcf, 0xa6, 0xa1, 0x6b, 0x60, 0x8b, 0x31, 0xcb, 0x03, 0xe5, 0x31, 0x78, 0x00, 0x8d, 0xb3,
0x94, 0xa4, 0x99, 0x78, 0x9c, 0x24, 0x3c, 0x41, 0x08, 0x9c, 0x63, 0x1e, 0x52, 0xc5, 0x58, 0xc6,
0xea, 0x8c, 0x3c, 0xa8, 0x3d, 0xa7, 0x42, 0x90, 0x01, 0xcd, 0xb3, 0x1b, 0x33, 0xf8, 0x60, 0x41,
0x03, 0xf3, 0x6b, 0x4c, 0xc5, 0x88, 0xc7, 0x82, 0xa2, 0x7f, 0xa0, 0x76, 0x49, 0x49, 0x48, 0x13,
0xe1, 0x59, 0xbe, 0xdd, 0x6c, 0xb4, 0xd0, 0x4e, 0x2e, 0xea, 0x98, 0xb3, 0x6c, 0x18, 0x9f, 0xc4,
0x7d, 0x8e, 0x0d, 0x05, 0xed, 0x41, 0xad, 0xa7, 0xdc, 0xc2, 0x5b, 0x50, 0xec, 0x8d, 0x32, 0xdb,
0xa4, 0xc5, 0x86, 0x86, 0xf6, 0x4b, 0xc5, 0x7a, 0xb6, 0x6f, 0x35, 0x1b, 0xad, 0x75, 0x13, 0x35,
0x03, 0xe1, 0x92, 0xa8, 0x4d, 0xa8, 0x87, 0x59, 0x42, 0xd2, 0x88, 0xc7, 0x9e, 0xe3, 0x5b, 0x4d,
0x1b, 0x17, 0x76, 0x70, 0x17, 0x6c, 0xcc, 0xaf, 0x67, 0x6b, 0xb1, 0x6e, 0x55, 0x4b, 0xf0, 0xce,
0x82, 0xe5, 0x97, 0xa4, 0xcb, 0xe8, 0x77, 0xaa, 0xff, 0x03, 0x9c, 0x84, 0x5f, 0x1b, 0xe9, 0x0d,
0x43, 0x95, 0xd7, 0xa9, 0x80, 0x1f, 0x21, 0xf6, 0x00, 0x60, 0x5a, 0x8a, 0xec, 0x75, 0x4c, 0x86,
0x34, 0x9f, 0x06, 0x75, 0x56, 0xd1, 0x24, 0x25, 0xe9, 0x64, 0x64, 0x9a, 0x5d, 0xd8, 0xc1, 0x5b,
0x1b, 0x56, 0xca, 0xb7, 0x81, 0x7e, 0x07, 0x57, 0xa4, 0x49, 0x14, 0x0f, 0x2e, 0x48, 0x3e, 0x55,
0xed, 0x0a, 0x9e, 0xba, 0x24, 0x9e, 0x45, 0x71, 0x7a, 0xe7, 0x7f, 0x89, 0xcb, 0x7c, 0x8e, 0xc4,
0x0b, 0x17, 0xfa, 0x0d, 0xea, 0x05, 0x2c, 0x05, 0xda, 0xed, 0x0a, 0x2e, 0x3c, 0x68, 0x13, 0x6a,
0x5d, 0xce, 0x99, 0x04, 0xa5, 0x92, 0x7a, 0xbb, 0x82, 0x8d, 0x43, 0x61, 0x8c, 0x77, 0x25, 0xb6,
0xe8, 0x5b, 0xcd, 0x25, 0x85, 0x69, 0x07, 0x7a, 0x08, 0x2b, 0xfa, 0x13, 0x87, 0x49, 0x42, 0x26,
0x92, 0x52, 0x2d, 0x5f, 0xde, 0xf9, 0x14, 0x6d, 0x57, 0xf0, 0x0d, 0xb2, 0x0c, 0xd7, 0x0a, 0x8a,
0xf0, 0xda, 0xcd, 0xbb, 0x2f, 0x50, 0x19, 0x5e, 0x26, 0x23, 0x1f, 0xa0, 0xcf, 0x38, 0xc9, 0x55,
0xd5, 0x7d, 0xab, 0x69, 0xb5, 0x2b, 0x78, 0xc6, 0x87, 0xfe, 0x05, 0x08, 0x69, 0x2f, 0x1a, 0x12,
0x25, 0xcd, 0x55, 0xc9, 0x57, 0x4d, 0xf2, 0x47, 0x1a, 0x91, 0x21, 0x53, 0xd2, 0x51, 0x03, 0x5c,
0x3d, 0x78, 0x17, 0x84, 0x05, 0xfb, 0x50, 0xcb, 0x59, 0x72, 0x17, 0x5c, 0x11, 0x96, 0xe9, 0x26,
0xda, 0x58, 0x1b, 0xd2, 0x2b, 0x7a, 0x84, 0xe9, 0x16, 0xda, 0x58, 0x1b, 0xc1, 0x7b, 0x0b, 0x56,
0x4e, 0x62, 0x31, 0xa2, 0xbd, 0xf4, 0xeb, 0xab, 0xe4, 0xef, 0xd9, 0x87, 0x29, 0x8b, 0xfb, 0xc9,
0x14, 0x77, 0x12, 0x8a, 0x17, 0xc9, 0x53, 0x3a, 0x11, 0xd3, 0x37, 0x19, 0xc0, 0x52, 0x3f, 0x62,
0x29, 0x4d, 0x9e, 0x44, 0x94, 0x85, 0xc2, 0xb3, 0x7d, 0xbb, 0xe9, 0xe2, 0x92, 0x4f, 0x7e, 0x86,
0x45, 0xc3, 0x28, 0x55, 0x6d, 0x74, 0xb0, 0x36, 0xd0, 0x06, 0x54, 0x79, 0xbf, 0x2f, 0x68, 0xaa,
0x3a, 0xe8, 0xe0, 0xdc, 0x92, 0xec, 0xb1, 0xdc, 0x5b, 0xaa, 0x6b, 0x2e, 0xd6, 0x46, 0xb0, 0x05,
0x8d, 0x99, 0xb6, 0xc9, 0xe1, 0xbd, 0x22, 0x4c, 0xbf, 0x34, 0x07, 0xab, 0xb3, 0xa4, 0xcc, 0xb4,
0xa6, 0x44, 0x71, 0x73, 0xca, 0x00, 0xdc, 0x42, 0x03, 0xfa, 0x13, 0xec, 0x28, 0x14, 0x4a, 0xfb,
0xdc, 0xe1, 0x90, 0x0c, 0xf4, 0x17, 0x38, 0xaf, 0xe9, 0xc4, 0xdc, 0xc6, 0x9c, 0x39, 0x50, 0x94,
0xa3, 0x2a, 0x38, 0xf2, 0xb1, 0xb4, 0x3e, 0x2e, 0x40, 0xb5, 0xa3, 0x68, 0xe8, 0x00, 0xea, 0x66,
0x0f, 0xa3, 0x5f, 0x4d, 0xec, 0x8d, 0xcd, 0xbc, 0xb9, 0x3e, 0xbb, 0x00, 0xf2, 0xe7, 0x15, 0x54,
0xf6, 0x2c, 0x74, 0x08, 0xcb, 0x86, 0x7b, 0x1e, 0x93, 0x64, 0x32, 0x3f, 0xc5, 0x2f, 0x06, 0x28,
0xad, 0xa5, 0xa0, 0x52, 0x14, 0xd0, 0xf9, 0xac, 0x80, 0xce, 0x37, 0x14, 0xd0, 0xf9, 0x72, 0x01,
0x9d, 0x5b, 0x14, 0x70, 0x1f, 0x6a, 0xf9, 0xe0, 0xa1, 0x62, 0xaf, 0x96, 0x27, 0x71, 0xee, 0xe7,
0x8f, 0xb6, 0x5f, 0x6d, 0x0d, 0xa2, 0xf4, 0x32, 0xeb, 0xee, 0xf4, 0xf8, 0x70, 0x57, 0x93, 0xcc,
0xcf, 0x55, 0x6b, 0x57, 0xfd, 0x5b, 0x76, 0xab, 0xea, 0xe7, 0xbf, 0x4f, 0x01, 0x00, 0x00, 0xff,
0xff, 0x18, 0x84, 0x1c, 0xb4, 0x44, 0x07, 0x00, 0x00,
// 923 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x56, 0xdb, 0x72, 0x1b, 0x45,
0x10, 0xd5, 0x66, 0xd7, 0xba, 0xf4, 0xfa, 0x96, 0x31, 0x31, 0x8b, 0x42, 0x81, 0x32, 0x2e, 0x2a,
0xa2, 0xa0, 0x9c, 0x20, 0x08, 0x29, 0xc0, 0x29, 0x2a, 0x76, 0x02, 0x72, 0x01, 0x85, 0x32, 0x21,
0x79, 0xe0, 0x6d, 0x24, 0x8d, 0x9c, 0x2d, 0x56, 0x3b, 0xf2, 0xce, 0xc8, 0x46, 0x3f, 0xc0, 0xff,
0xf0, 0xcc, 0x0b, 0xdf, 0xc2, 0x97, 0x50, 0x73, 0xdb, 0x8b, 0x2e, 0x24, 0x50, 0xc5, 0x93, 0xa6,
0xfb, 0x9c, 0xee, 0xe9, 0xd3, 0x3d, 0x33, 0x5a, 0xd8, 0x9e, 0xc5, 0x09, 0x17, 0xf4, 0x78, 0x96,
0x71, 0xc9, 0x51, 0xdd, 0x58, 0xf8, 0x0b, 0xd8, 0x7b, 0x36, 0x67, 0xd9, 0x62, 0xf0, 0xec, 0x7b,
0xc2, 0x2e, 0xe7, 0x4c, 0x48, 0xf4, 0x16, 0x6c, 0xc5, 0xe9, 0x98, 0xfd, 0x1a, 0x79, 0x1d, 0xaf,
0xdb, 0x22, 0xc6, 0x40, 0xfb, 0xe0, 0xcf, 0x2e, 0x93, 0xe8, 0x86, 0xf6, 0xa9, 0x25, 0x3e, 0xb2,
0xa1, 0xcf, 0x8b, 0xd0, 0x7d, 0xf0, 0xc5, 0x65, 0x62, 0x03, 0xd5, 0x12, 0x7f, 0x05, 0xe1, 0x73,
0x49, 0xe5, 0x5c, 0x3c, 0xcd, 0x32, 0x9e, 0x21, 0x04, 0xc1, 0x19, 0x1f, 0x33, 0xcd, 0xd8, 0x21,
0x7a, 0x8d, 0x22, 0x68, 0xfc, 0xc0, 0x84, 0xa0, 0x17, 0xcc, 0x66, 0x77, 0x26, 0xfe, 0xd3, 0x83,
0x90, 0xf0, 0x6b, 0xc2, 0xc4, 0x8c, 0xa7, 0x82, 0xa1, 0x8f, 0xa1, 0xf1, 0x8a, 0xd1, 0x31, 0xcb,
0x44, 0xe4, 0x75, 0xfc, 0x6e, 0xd8, 0x43, 0xc7, 0x56, 0xd4, 0x19, 0x4f, 0xe6, 0xd3, 0xf4, 0x3c,
0x9d, 0x70, 0xe2, 0x28, 0xe8, 0x3e, 0x34, 0x46, 0xda, 0x2d, 0xa2, 0x1b, 0x9a, 0x7d, 0x58, 0x65,
0xbb, 0xb4, 0xc4, 0xd1, 0xd0, 0x83, 0x4a, 0xb1, 0x91, 0xdf, 0xf1, 0xba, 0x61, 0xef, 0xc0, 0x45,
0x95, 0x20, 0x52, 0x11, 0xd5, 0x86, 0xe6, 0x78, 0x9e, 0x51, 0x19, 0xf3, 0x34, 0x0a, 0x3a, 0x5e,
0xd7, 0x27, 0xb9, 0x8d, 0x1f, 0x82, 0x4f, 0xf8, 0x75, 0xb9, 0x16, 0xef, 0x8d, 0x6a, 0xc1, 0xbf,
0x7b, 0xb0, 0xf3, 0x13, 0x1d, 0x26, 0xec, 0x3f, 0xaa, 0x7f, 0x1f, 0x82, 0x8c, 0x5f, 0x3b, 0xe9,
0xa1, 0xa3, 0xaa, 0x76, 0x6a, 0xe0, 0xff, 0x10, 0x7b, 0x02, 0x50, 0x94, 0xa2, 0x66, 0x9d, 0xd2,
0x29, 0xb3, 0xa7, 0x41, 0xaf, 0x75, 0x34, 0x95, 0x54, 0x2e, 0x66, 0x6e, 0xd8, 0xb9, 0x8d, 0x7f,
0xf3, 0x61, 0xb7, 0xda, 0x0d, 0xf4, 0x1e, 0xb4, 0x84, 0xcc, 0xe2, 0xf4, 0xe2, 0x25, 0xb5, 0xa7,
0xaa, 0x5f, 0x23, 0x85, 0x4b, 0xe1, 0xf3, 0x38, 0x95, 0x9f, 0x7f, 0xa6, 0x70, 0x95, 0x2f, 0x50,
0x78, 0xee, 0x42, 0xef, 0x42, 0x33, 0x87, 0x95, 0x40, 0xbf, 0x5f, 0x23, 0xb9, 0x07, 0xb5, 0xa1,
0x31, 0xe4, 0x3c, 0x51, 0xa0, 0x52, 0xd2, 0xec, 0xd7, 0x88, 0x73, 0x68, 0x2c, 0xe1, 0x43, 0x85,
0x6d, 0x75, 0xbc, 0xee, 0xb6, 0xc6, 0x8c, 0x03, 0x3d, 0x82, 0x5d, 0xb3, 0xc5, 0xe3, 0x2c, 0xa3,
0x0b, 0x45, 0xa9, 0x57, 0x9b, 0xf7, 0xa2, 0x40, 0xfb, 0x35, 0xb2, 0x44, 0x56, 0xe1, 0x46, 0x41,
0x1e, 0xde, 0x58, 0xee, 0x7d, 0x8e, 0xaa, 0xf0, 0x2a, 0x19, 0x75, 0x00, 0x26, 0x09, 0xa7, 0x56,
0x55, 0xb3, 0xe3, 0x75, 0xbd, 0x7e, 0x8d, 0x94, 0x7c, 0xe8, 0x13, 0x80, 0x31, 0x1b, 0xc5, 0x53,
0xaa, 0xa5, 0xb5, 0x74, 0xf2, 0x3d, 0x97, 0xfc, 0x89, 0x41, 0x54, 0x48, 0x41, 0x3a, 0x0d, 0xa1,
0x65, 0x0e, 0xde, 0x4b, 0x9a, 0xe0, 0x07, 0xd0, 0xb0, 0x2c, 0xf5, 0x16, 0x5c, 0xd1, 0x64, 0x6e,
0x86, 0xe8, 0x13, 0x63, 0x28, 0xaf, 0x18, 0xd1, 0xc4, 0x8c, 0xd0, 0x27, 0xc6, 0xc0, 0x7f, 0x78,
0xb0, 0x7b, 0x9e, 0x8a, 0x19, 0x1b, 0xc9, 0x7f, 0x7e, 0x4a, 0x3e, 0x2a, 0x5f, 0x4c, 0x55, 0xdc,
0x4d, 0x57, 0xdc, 0xf9, 0x58, 0xfc, 0x98, 0x7d, 0xc7, 0x16, 0xa2, 0xb8, 0x93, 0x18, 0xb6, 0x27,
0x71, 0x22, 0x59, 0xf6, 0x4d, 0xcc, 0x92, 0xb1, 0x88, 0xfc, 0x8e, 0xdf, 0x6d, 0x91, 0x8a, 0x4f,
0x6d, 0x93, 0xc4, 0xd3, 0x58, 0xea, 0x31, 0x06, 0xc4, 0x18, 0xe8, 0x10, 0xea, 0x7c, 0x32, 0x11,
0x4c, 0xea, 0x09, 0x06, 0xc4, 0x5a, 0x8a, 0x7d, 0xa9, 0xde, 0x2d, 0x3d, 0xb5, 0x16, 0x31, 0x06,
0xbe, 0x03, 0x61, 0x69, 0x6c, 0xea, 0xf0, 0x5e, 0xd1, 0xc4, 0xdc, 0xb4, 0x80, 0xe8, 0xb5, 0xa2,
0x94, 0x46, 0x53, 0xa1, 0xb4, 0x2c, 0xe5, 0x02, 0x5a, 0xb9, 0x06, 0x74, 0x17, 0xfc, 0x78, 0x2c,
0xb4, 0xf6, 0x8d, 0x87, 0x43, 0x31, 0xd0, 0x87, 0x10, 0xfc, 0xc2, 0x16, 0xae, 0x1b, 0x1b, 0xce,
0x81, 0xa6, 0x9c, 0xd6, 0x21, 0xd0, 0x97, 0xe5, 0x36, 0x6c, 0x9d, 0xeb, 0x66, 0xae, 0xb9, 0x65,
0xf8, 0x04, 0xd0, 0x59, 0xc6, 0xa8, 0x64, 0x9a, 0xe2, 0x86, 0xb1, 0xee, 0x3e, 0xa2, 0xd2, 0xce,
0x4d, 0xb3, 0x05, 0xbe, 0x05, 0x07, 0x95, 0x68, 0x73, 0x17, 0xf1, 0x07, 0xb0, 0xf7, 0x2d, 0x93,
0xaf, 0xcb, 0x88, 0x1f, 0xc2, 0x7e, 0x41, 0xb3, 0xd7, 0xf8, 0xa8, 0x7c, 0x0c, 0xc2, 0xde, 0x4e,
0x3e, 0x6e, 0xcd, 0x32, 0x18, 0x3e, 0x80, 0x9b, 0x2e, 0x90, 0x09, 0xbb, 0x03, 0x7e, 0x04, 0xa8,
0xec, 0xb4, 0xf9, 0xee, 0x42, 0x23, 0x36, 0x2e, 0xfb, 0x12, 0x2e, 0x65, 0x74, 0x28, 0xee, 0x02,
0x7a, 0xc2, 0x12, 0xf6, 0xfa, 0x46, 0x28, 0xd1, 0x15, 0xa6, 0xd9, 0xa9, 0xf7, 0x57, 0x00, 0xf5,
0x81, 0x4e, 0x8d, 0xfa, 0x10, 0x96, 0xda, 0x82, 0xda, 0xf9, 0xe3, 0xbb, 0xd2, 0xe9, 0xf6, 0xed,
0xb5, 0x98, 0xed, 0x63, 0x0d, 0x3d, 0x05, 0x28, 0x44, 0xa1, 0x77, 0x1c, 0x79, 0x45, 0x7d, 0xbb,
0xbd, 0x0e, 0xca, 0xd3, 0x7c, 0x0d, 0x4d, 0xe7, 0x47, 0x6f, 0x2f, 0x33, 0x5d, 0x8a, 0x68, 0x15,
0xc8, 0x13, 0xf4, 0x21, 0x2c, 0x69, 0x2e, 0x14, 0xad, 0xb6, 0xac, 0x50, 0xb4, 0xa6, 0x49, 0xb8,
0x86, 0x4e, 0xa0, 0xe9, 0x3e, 0x05, 0x8a, 0x52, 0x96, 0x3e, 0x0e, 0xda, 0x07, 0xe5, 0xff, 0xa0,
0x3c, 0xf6, 0xbe, 0x87, 0x1e, 0xc3, 0x8e, 0xe3, 0xbe, 0x48, 0x69, 0xb6, 0xd8, 0x9c, 0xe2, 0x96,
0x03, 0x2a, 0xff, 0x8c, 0xa5, 0x02, 0x06, 0x2b, 0x05, 0x0c, 0xfe, 0x45, 0x01, 0x83, 0xf5, 0x05,
0x0c, 0xde, 0xa0, 0x80, 0x2f, 0xa1, 0x61, 0xdf, 0x3e, 0x74, 0x58, 0x1c, 0xc6, 0xf2, 0x63, 0xb8,
0x71, 0xfb, 0xd3, 0xa3, 0x9f, 0xef, 0x5c, 0xc4, 0xf2, 0xd5, 0x7c, 0x78, 0x3c, 0xe2, 0xd3, 0x7b,
0x86, 0xe4, 0x7e, 0xae, 0x7a, 0xf7, 0xf4, 0x07, 0xdb, 0xb0, 0xae, 0x7f, 0x3e, 0xfd, 0x3b, 0x00,
0x00, 0xff, 0xff, 0x75, 0x3f, 0x36, 0x94, 0xc7, 0x09, 0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.
@ -920,6 +1274,10 @@ const _ = grpc.SupportPackageIsVersion4
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
type PilosaClient interface {
CreateIndex(ctx context.Context, in *CreateIndexRequest, opts ...grpc.CallOption) (*CreateIndexResponse, error)
GetIndexes(ctx context.Context, in *GetIndexesRequest, opts ...grpc.CallOption) (*GetIndexesResponse, error)
GetIndex(ctx context.Context, in *GetIndexRequest, opts ...grpc.CallOption) (*GetIndexResponse, error)
DeleteIndex(ctx context.Context, in *DeleteIndexRequest, opts ...grpc.CallOption) (*DeleteIndexResponse, error)
QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Pilosa_QuerySQLClient, error)
QuerySQLUnary(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (*TableResponse, error)
QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Pilosa_QueryPQLClient, error)
@ -935,6 +1293,42 @@ func NewPilosaClient(cc *grpc.ClientConn) PilosaClient {
return &pilosaClient{cc}
}
func (c *pilosaClient) CreateIndex(ctx context.Context, in *CreateIndexRequest, opts ...grpc.CallOption) (*CreateIndexResponse, error) {
out := new(CreateIndexResponse)
err := c.cc.Invoke(ctx, "/pilosa.Pilosa/CreateIndex", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *pilosaClient) GetIndexes(ctx context.Context, in *GetIndexesRequest, opts ...grpc.CallOption) (*GetIndexesResponse, error) {
out := new(GetIndexesResponse)
err := c.cc.Invoke(ctx, "/pilosa.Pilosa/GetIndexes", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *pilosaClient) GetIndex(ctx context.Context, in *GetIndexRequest, opts ...grpc.CallOption) (*GetIndexResponse, error) {
out := new(GetIndexResponse)
err := c.cc.Invoke(ctx, "/pilosa.Pilosa/GetIndex", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *pilosaClient) DeleteIndex(ctx context.Context, in *DeleteIndexRequest, opts ...grpc.CallOption) (*DeleteIndexResponse, error) {
out := new(DeleteIndexResponse)
err := c.cc.Invoke(ctx, "/pilosa.Pilosa/DeleteIndex", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *pilosaClient) QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Pilosa_QuerySQLClient, error) {
stream, err := c.cc.NewStream(ctx, &_Pilosa_serviceDesc.Streams[0], "/pilosa.Pilosa/QuerySQL", opts...)
if err != nil {
@ -1051,6 +1445,10 @@ func (x *pilosaInspectClient) Recv() (*RowResponse, error) {
// PilosaServer is the server API for Pilosa service.
type PilosaServer interface {
CreateIndex(context.Context, *CreateIndexRequest) (*CreateIndexResponse, error)
GetIndexes(context.Context, *GetIndexesRequest) (*GetIndexesResponse, error)
GetIndex(context.Context, *GetIndexRequest) (*GetIndexResponse, error)
DeleteIndex(context.Context, *DeleteIndexRequest) (*DeleteIndexResponse, error)
QuerySQL(*QuerySQLRequest, Pilosa_QuerySQLServer) error
QuerySQLUnary(context.Context, *QuerySQLRequest) (*TableResponse, error)
QueryPQL(*QueryPQLRequest, Pilosa_QueryPQLServer) error
@ -1062,6 +1460,18 @@ type PilosaServer interface {
type UnimplementedPilosaServer struct {
}
func (*UnimplementedPilosaServer) CreateIndex(ctx context.Context, req *CreateIndexRequest) (*CreateIndexResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method CreateIndex not implemented")
}
func (*UnimplementedPilosaServer) GetIndexes(ctx context.Context, req *GetIndexesRequest) (*GetIndexesResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetIndexes not implemented")
}
func (*UnimplementedPilosaServer) GetIndex(ctx context.Context, req *GetIndexRequest) (*GetIndexResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetIndex not implemented")
}
func (*UnimplementedPilosaServer) DeleteIndex(ctx context.Context, req *DeleteIndexRequest) (*DeleteIndexResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method DeleteIndex not implemented")
}
func (*UnimplementedPilosaServer) QuerySQL(req *QuerySQLRequest, srv Pilosa_QuerySQLServer) error {
return status.Errorf(codes.Unimplemented, "method QuerySQL not implemented")
}
@ -1082,6 +1492,78 @@ func RegisterPilosaServer(s *grpc.Server, srv PilosaServer) {
s.RegisterService(&_Pilosa_serviceDesc, srv)
}
func _Pilosa_CreateIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(CreateIndexRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(PilosaServer).CreateIndex(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/pilosa.Pilosa/CreateIndex",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(PilosaServer).CreateIndex(ctx, req.(*CreateIndexRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Pilosa_GetIndexes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetIndexesRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(PilosaServer).GetIndexes(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/pilosa.Pilosa/GetIndexes",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(PilosaServer).GetIndexes(ctx, req.(*GetIndexesRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Pilosa_GetIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetIndexRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(PilosaServer).GetIndex(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/pilosa.Pilosa/GetIndex",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(PilosaServer).GetIndex(ctx, req.(*GetIndexRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Pilosa_DeleteIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(DeleteIndexRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(PilosaServer).DeleteIndex(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/pilosa.Pilosa/DeleteIndex",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(PilosaServer).DeleteIndex(ctx, req.(*DeleteIndexRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Pilosa_QuerySQL_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(QuerySQLRequest)
if err := stream.RecvMsg(m); err != nil {
@ -1185,6 +1667,22 @@ var _Pilosa_serviceDesc = grpc.ServiceDesc{
ServiceName: "pilosa.Pilosa",
HandlerType: (*PilosaServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "CreateIndex",
Handler: _Pilosa_CreateIndex_Handler,
},
{
MethodName: "GetIndexes",
Handler: _Pilosa_GetIndexes_Handler,
},
{
MethodName: "GetIndex",
Handler: _Pilosa_GetIndex_Handler,
},
{
MethodName: "DeleteIndex",
Handler: _Pilosa_DeleteIndex_Handler,
},
{
MethodName: "QuerySQLUnary",
Handler: _Pilosa_QuerySQLUnary_Handler,

38
proto/pilosa.proto
View file

@ -85,7 +85,45 @@ message IdsOrKeys {
}
}
message Index {
string name = 1;
}
message CreateIndexRequest {
string name = 1;
bool keys = 2;
}
message CreateIndexResponse {
}
message GetIndexRequest {
string name = 1;
}
message GetIndexResponse {
Index index = 1;
}
message GetIndexesRequest {
}
message GetIndexesResponse {
repeated Index indexes = 1;
}
message DeleteIndexRequest {
string name = 1;
}
message DeleteIndexResponse {
}
service Pilosa {
rpc CreateIndex(CreateIndexRequest) returns (CreateIndexResponse) {};
rpc GetIndexes(GetIndexesRequest) returns (GetIndexesResponse) {};
rpc GetIndex(GetIndexRequest) returns (GetIndexResponse) {};
rpc DeleteIndex(DeleteIndexRequest) returns (DeleteIndexResponse) {};
rpc QuerySQL(QuerySQLRequest) returns (stream RowResponse) {};
rpc QuerySQLUnary(QuerySQLRequest) returns (TableResponse) {};
rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {};

17
rbf/cursor.go
View file

@ -1386,21 +1386,8 @@ func (c *Cursor) difference(key uint64, data *roaring.Container) (bool, error) {
}
func (c *Cursor) Close() {
if c == nil {
panic("cannot Close nil Cursor")
}
tx := c.tx
c.tx = nil // allow tx to be garbage collected.
if tx.db.cfg.CursorCacheSize == 0 {
globalCursorSyncPool.Put(c)
return
}
select {
case tx.db.cursorArenaCh <- c:
case <-tx.db.cursorCleaner.ReqStop.Chan:
}
assert(c != nil)
cursorSyncPool.Put(c)
}
func keysFromParents(parents []branchCell) (ckeys []int) {

40
rbf/db.go
View file

@ -24,7 +24,6 @@ import (
"syscall"
"github.com/benbjohnson/immutable"
"github.com/glycerine/idem"
rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg"
"github.com/pilosa/pilosa/v2/syswrap"
)
@ -33,9 +32,9 @@ var (
ErrClosed = errors.New("rbf: database closed")
)
// global in the sense that it is shared among all instances
// of rbf.DBs in this process. This is deliberate.
var globalCursorSyncPool = &sync.Pool{
// shared cursor pool across all DB instances.
// Cursors are returned on Cursor.Close().
var cursorSyncPool = &sync.Pool{
New: func() interface{} {
return &Cursor{}
},
@ -63,9 +62,6 @@ type DB struct {
// Path represents the path to the database file.
Path string
cursorArenaCh chan *Cursor
cursorCleaner *idem.Halter
}
// NewDB returns a new instance of DB.
@ -79,12 +75,6 @@ func NewDB(path string, cfg *rbfcfg.Config) *DB {
txs: make(map[*Tx]struct{}),
pageMap: NewPageMap(),
Path: path,
cursorArenaCh: make(chan *Cursor, cfg.CursorCacheSize),
cursorCleaner: idem.NewHalter(),
}
for i := int64(0); i < cfg.CursorCacheSize; i++ {
db.cursorArenaCh <- &Cursor{}
}
db.haltCond = sync.NewCond(&db.mu)
@ -272,8 +262,6 @@ func (db *DB) Close() (err error) {
db.mu.Lock()
defer db.mu.Unlock()
defer db.cursorCleaner.RequestStop()
db.opened = false
// Close mmap handle.
@ -580,24 +568,10 @@ func (db *DB) readMetaPage() ([]byte, error) {
return db.readDBPage(0)
}
func (db *DB) getCursor(tx *Tx) (c *Cursor) {
if db.cfg.CursorCacheSize == 0 {
c = globalCursorSyncPool.Get().(*Cursor)
c.tx = tx
return
}
n := len(db.cursorArenaCh)
if n < 10 {
vv("warning, db.cursorArenaCh is low! %v left", n)
}
select {
case c = <-db.cursorArenaCh:
c.tx = tx
return
case <-db.cursorCleaner.ReqStop.Chan:
return nil
}
func (db *DB) getCursor(tx *Tx) *Cursor {
c := cursorSyncPool.Get().(*Cursor)
c.tx = tx
return c
}
// Shared pool for in-memory database pages.

4
rrtx.go
View file

@ -250,7 +250,7 @@ func (tx *RoaringTx) Contains(index, field, view string, shard uint64, v uint64)
func (tx *RoaringTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, false, err
return nil, false, errors.Wrap(err, "getting bitmap")
}
//vv("b bitmap back from bitmap(index='%v', field='%v', view='%v', shard='%v')='%#v'", index, field, view, shard, b.Slice())
citer, found = b.Containers.Iterator(key)
@ -395,7 +395,7 @@ const FragmentNotFound = Error("fragment not found")
func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
frag, err := tx.getFragment(index, field, view, shard)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "getFragment")
}
return frag.storage, nil
}

42
scripts/bench_read.sh Executable file
View file

@ -0,0 +1,42 @@
#!/bin/bash -x
set -e
# This script runs benchmarks and posts them to Slack's #nightly channel.
# The caller of the script should `git pull` on the pilosa repo before executing.
#
# Environment variables:
# - PILOSA_SRC: Path to pilosa src directory.
# - SLACK_OAUTH_TOKEN: Token used to post to Slack.
# Require environment variables.
: "${PILOSA_SRC:?Must set PILOSA_SRC environment variable}"
: "${SLACK_OAUTH_TOKEN:?Must set SLACK_OAUTH_TOKEN environment variable}"
# Build pilosa into GOBIN.
make -C $PILOSA_SRC install install-bench
# Retrieve current SHA.
SHA=$(git -C $PILOSA_SRC rev-parse HEAD)
# Format current date.
DATE=$(date '+%Y%m%d')
for TYPE in row row-bsi row-range count intersect union difference xor groupby topk
do
WORKFLOW_PATH="${BASH_SOURCE%/*}/etc/gloat/query.${TYPE}.yml"
WORKFLOW_NAME="$(gloat workflow name $WORKFLOW_PATH)"
TITLE="$WORKFLOW_NAME, $DATE ($SHA)"
# Execute RBF/Roaring benchmark.
RBF_PATH=gloat/data/query/${TYPE}/rbf/${DATE}.tar.gz
TXSRC=rbf gloat run -v -o "$RBF_PATH" $WORKFLOW_PATH
ROARING_PATH=gloat/data/query/${TYPE}/roaring/${DATE}.tar.gz
TXSRC=roaring gloat run -v -o "$ROARING_PATH" $WORKFLOW_PATH
# Generate graph from results.
gloat graph -layout 2,5 -size 5120,820 -title "$TITLE" -name utime,stime,heap_alloc,heap_inuse,heap_objects,num_gc,rchar,wchar,syscr,syscw -series rbf,roaring -o /tmp/output.png $RBF_PATH $ROARING_PATH
# Post graph to Slack with SHA.
curl -F file=@/tmp/output.png -F channels=C01HBFKRLGH -F "initial_comment=$TITLE" -H "Authorization: Bearer $SLACK_OAUTH_TOKEN" https://slack.com/api/files.upload
done

3
scripts/bench.sh → scripts/bench_write.sh
View file

@ -1,4 +1,5 @@
#!/bin/bash
set -e
# This script runs benchmarks and posts them to Slack's #nightly channel.
# The caller of the script should `git pull` on the pilosa repo before executing.
@ -32,7 +33,7 @@ ROARING_PATH=gloat/data/1m/roaring/${DATE}.tar.gz
TXSRC=roaring gloat run -v -o $ROARING_PATH $WORKFLOW_PATH
# Generate graph from results.
gloat graph -layout 5,2 -size 2048,2048 -title "$TITLE" -name utime,stime,heap_alloc,heap_inuse,heap_objects,num_gc,rchar,wchar,syscr,syscw -series rbf,roaring -o /tmp/output.png $RBF_PATH $ROARING_PATH
gloat graph -layout 2,5 -size 5120,820 -title "$TITLE" -name utime,stime,heap_alloc,heap_inuse,heap_objects,num_gc,rchar,wchar,syscr,syscw -series rbf,roaring -o /tmp/output.png $RBF_PATH $ROARING_PATH
# Post graph to Slack with SHA.
curl -F file=@/tmp/output.png -F channels=C01HBFKRLGH -F "initial_comment=$TITLE" -H "Authorization: Bearer $SLACK_OAUTH_TOKEN" https://slack.com/api/files.upload

6
scripts/etc/gloat/gh.1d.yml
View file

@ -1,7 +1,9 @@
name: "GitHub Import Load Testing (1 day)"
main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}"
load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-01T23:00:00Z --cache-dir .githubarchive"
load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-01T23:00:00Z --cache-dir ~/.githubarchive"
health_url: "http://localhost:10101/status"
debug_url: "http://localhost:10101/debug"
vars_urls:
- http://localhost:10101/debug/vars
- http://localhost:7070/debug/vars

6
scripts/etc/gloat/gh.1m.yml
View file

@ -1,7 +1,9 @@
name: "GitHub Import Load Testing (1 month)"
main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}"
load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-31T23:00:00Z --cache-dir .githubarchive"
load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-31T23:00:00Z --cache-dir ~/.githubarchive"
health_url: "http://localhost:10101/status"
debug_url: "http://localhost:10101/debug"
vars_urls:
- http://localhost:10101/debug/vars
- http://localhost:7070/debug/vars

6
scripts/etc/gloat/gh.1w.yml
View file

@ -1,7 +1,9 @@
name: "GitHub Import Load Testing (1 week)"
main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}"
load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-06T23:00:00Z --cache-dir .githubarchive"
load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-06T23:00:00Z --cache-dir ~/.githubarchive"
health_url: "http://localhost:10101/status"
debug_url: "http://localhost:10101/debug"
vars_urls:
- http://localhost:10101/debug/vars
- http://localhost:7070/debug/vars

10
scripts/etc/gloat/query.count.yml Normal file
View file

@ -0,0 +1,10 @@
name: "Count() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
load: "pilosa-bench -type count -rate 100 -n 3000"
health_url: "http://localhost:10101/status"
health_regexp: "NORMAL"
vars_urls:
- http://localhost:10101/debug/vars

10
scripts/etc/gloat/query.difference.yml Normal file
View file

@ -0,0 +1,10 @@
name: "Difference() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
load: "pilosa-bench -type difference -rate 10 -n 300"
health_url: "http://localhost:10101/status"
health_regexp: "NORMAL"
vars_urls:
- http://localhost:10101/debug/vars

10
scripts/etc/gloat/query.groupby.yml Normal file
View file

@ -0,0 +1,10 @@
name: "GroupBy() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
load: "pilosa-bench -type groupby -rate 100 -n 3000"
health_url: "http://localhost:10101/status"
health_regexp: "NORMAL"
vars_urls:
- http://localhost:10101/debug/vars

10
scripts/etc/gloat/query.intersect.yml Normal file
View file

@ -0,0 +1,10 @@
name: "Intersect() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
load: "pilosa-bench -type intersect -rate 100 -n 3000"
health_url: "http://localhost:10101/status"
health_regexp: "NORMAL"
vars_urls:
- http://localhost:10101/debug/vars

10
scripts/etc/gloat/query.row-bsi.yml Normal file
View file

@ -0,0 +1,10 @@
name: "Row(BSI) Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
load: "pilosa-bench -type row -rate 100 -n 3000"
health_url: "http://localhost:10101/status"
health_regexp: "NORMAL"
vars_urls:
- http://localhost:10101/debug/vars

10
scripts/etc/gloat/query.row-range.yml Normal file
View file

@ -0,0 +1,10 @@
name: "Time-based Row() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
load: "pilosa-bench -type row-range -rate 10 -n 300 -from 2020-01-01T00:00:00Z -to 2020-01-31T23:00:00Z"
health_url: "http://localhost:10101/status"
health_regexp: "NORMAL"
vars_urls:
- http://localhost:10101/debug/vars

10
scripts/etc/gloat/query.row.yml Normal file
View file

@ -0,0 +1,10 @@
name: "Row() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
load: "pilosa-bench -type row -rate 100 -n 3000"
health_url: "http://localhost:10101/status"
health_regexp: "NORMAL"
vars_urls:
- http://localhost:10101/debug/vars

10
scripts/etc/gloat/query.topk.yml Normal file
View file

@ -0,0 +1,10 @@
name: "Time-based TopK() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
load: "pilosa-bench -type row-range -rate 10 -n 300 -from 2020-01-01T00:00:00Z -to 2020-01-31T23:00:00Z"
health_url: "http://localhost:10101/status"
health_regexp: "NORMAL"
vars_urls:
- http://localhost:10101/debug/vars

10
scripts/etc/gloat/query.union.yml Normal file
View file

@ -0,0 +1,10 @@
name: "Union() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
load: "pilosa-bench -type union -rate 10 -n 300"
health_url: "http://localhost:10101/status"
health_regexp: "NORMAL"
vars_urls:
- http://localhost:10101/debug/vars

10
scripts/etc/gloat/query.xor.yml Normal file
View file

@ -0,0 +1,10 @@
name: "Xor() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
load: "pilosa-bench -type xor -rate 10 -n 300"
health_url: "http://localhost:10101/status"
health_regexp: "NORMAL"
vars_urls:
- http://localhost:10101/debug/vars

28
scripts/populate_query_db.sh Executable file
View file

@ -0,0 +1,28 @@
#!/bin/bash
set -e
# This script generates data query load testing to be run against.
#
# Environment variables:
# - TXSRC: Transaction store type ("roaring", "rbf")
# - CACHEDIR: Path to local GitHub Archive data, if available.
# Require environment variables.
: "${TXSRC:?Must set TXSRC environment variable}"
: "${GHCACHEDIR:''}"
echo "Starting pilosa"
pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC} & pid_pilosa=$!
sleep 5
echo ""
echo "Importing GitHub Archive"
molecula-consumer-github -i events -d id --record-type event --batch-size=100000 \
--start-time 2020-01-01T00:00:00Z --end-time 2020-01-31T23:00:00Z \
--cache-dir "$GHCACHEDIR"
echo ""
echo "Import complete, shutting down pilosa"
sleep 5
kill $pid_pilosa

53
server/grpc.go
View file

@ -68,7 +68,7 @@ func errToStatusError(err error) error {
}
// Check error string.
switch errors.Cause(err) {
switch cause := errors.Cause(err); cause {
case pilosa.ErrIndexNotFound,
pilosa.ErrFieldNotFound,
pilosa.ErrForeignIndexNotFound,
@ -123,6 +123,10 @@ func errToStatusError(err error) error {
pilosa.ErrTooManyWrites,
pilosa.ErrNodeIDNotExists:
return status.Error(codes.Internal, err.Error())
default:
if _, ok := cause.(pilosa.ConflictError); ok {
return status.Error(codes.AlreadyExists, err.Error())
}
}
return status.Error(codes.Unknown, err.Error())
@ -267,6 +271,47 @@ func (h *GRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest
return table, errToStatusError(nil)
}
// CreateIndex creates a new Index
func (h *GRPCHandler) CreateIndex(ctx context.Context, req *pb.CreateIndexRequest) (*pb.CreateIndexResponse, error) {
// Always enable TrackExistence for gRPC-created indexes
opts := pilosa.IndexOptions{Keys: req.Keys, TrackExistence: true}
_, err := h.api.CreateIndex(ctx, req.Name, opts)
if err != nil {
return nil, errToStatusError(err)
}
return &pb.CreateIndexResponse{}, nil
}
// GetIndex returns a single Index given a name
func (h *GRPCHandler) GetIndex(ctx context.Context, req *pb.GetIndexRequest) (*pb.GetIndexResponse, error) {
schema := h.api.Schema(ctx)
for _, index := range schema {
if req.Name == index.Name {
return &pb.GetIndexResponse{Index: &pb.Index{Name: index.Name}}, nil
}
}
return nil, status.Error(codes.NotFound, fmt.Sprintf("Index with name %s not found", req.Name))
}
// GetIndexes returns a list of all Indexes
func (h *GRPCHandler) GetIndexes(ctx context.Context, req *pb.GetIndexesRequest) (*pb.GetIndexesResponse, error) {
schema := h.api.Schema(ctx)
indexes := make([]*pb.Index, len(schema))
for i, index := range schema {
indexes[i] = &pb.Index{Name: index.Name}
}
return &pb.GetIndexesResponse{Indexes: indexes}, nil
}
// DeleteIndex deletes an Index
func (h *GRPCHandler) DeleteIndex(ctx context.Context, req *pb.DeleteIndexRequest) (*pb.DeleteIndexResponse, error) {
err := h.api.DeleteIndex(ctx, req.Name)
if err != nil {
return nil, errToStatusError(err)
}
return &pb.DeleteIndexResponse{}, nil
}
// VDSMGRPCHandler contains methods which handle the various gRPC requests, ported from VDSM.
type VDSMGRPCHandler struct {
grpcHandler *GRPCHandler
@ -411,7 +456,8 @@ func ToTablerWrapper(result interface{}) (pb.ToTabler, error) {
if !ok {
switch v := result.(type) {
case []pilosa.GroupCount:
toTabler = pilosa.GroupCounts(v)
gc := pilosa.NewGroupCounts("", v...)
toTabler = gc
case uint64:
toTabler = ResultUint64(v)
case bool:
@ -432,7 +478,8 @@ func ToRowserWrapper(result interface{}) (pb.ToRowser, error) {
if !ok {
switch v := result.(type) {
case []pilosa.GroupCount:
toRowser = pilosa.GroupCounts(v)
gc := pilosa.NewGroupCounts("", v...)
toRowser = gc
case uint64:
toRowser = ResultUint64(v)
case bool:

196
server/grpc_test.go
View file

@ -138,7 +138,7 @@ func TestGRPC(t *testing.T) {
},
// []GroupCount (uint64)
{
[]pilosa.GroupCount{
pilosa.NewGroupCounts("", []pilosa.GroupCount{
pilosa.GroupCount{
Group: []pilosa.FieldRow{
{Field: "a", RowID: 10},
@ -160,22 +160,21 @@ func TestGRPC(t *testing.T) {
},
Count: 789,
},
},
}...),
[]expHeader{
{"a", "uint64"},
{"b", "uint64"},
{"count", "uint64"},
{"sum", "int64"},
},
[][]expColumn{
{uint64(10), uint64(11), uint64(123), int64(0)},
{uint64(10), uint64(12), uint64(456), int64(0)},
{int64(va), int64(vb), uint64(789), int64(0)},
{uint64(10), uint64(11), uint64(123)},
{uint64(10), uint64(12), uint64(456)},
{int64(va), int64(vb), uint64(789)},
},
},
// []GroupCount (string)
// []GroupCount (string) + sum
{
[]pilosa.GroupCount{
pilosa.NewGroupCounts("sum", []pilosa.GroupCount{
pilosa.GroupCount{
Group: []pilosa.FieldRow{
{Field: "a", RowKey: "ten"},
@ -190,7 +189,7 @@ func TestGRPC(t *testing.T) {
},
Count: 456,
},
},
}...),
[]expHeader{
{"a", "string"},
{"b", "string"},
@ -292,7 +291,12 @@ func TestGRPC(t *testing.T) {
}
// Ensure headers match.
for i, header := range table.GetHeaders() {
headers := table.GetHeaders()
if len(headers) < len(test.expHeaders) {
t.Fatalf("test %d expected %d headers, got %d, first missing header %q",
ti, len(test.expHeaders), len(headers), test.expHeaders[len(headers)].name)
}
for i, header := range headers {
if header.Name != test.expHeaders[i].name {
t.Fatalf("test %d expected header name: %s, but got: %s", ti, test.expHeaders[i].name, header.Name)
}
@ -303,7 +307,12 @@ func TestGRPC(t *testing.T) {
// Ensure column data matches.
for i, row := range table.GetRows() {
for j, column := range row.GetColumns() {
columns := row.GetColumns()
if len(columns) != len(test.expColumns[i]) {
t.Fatalf("test %d expected %d columns, got %d in row %d",
ti, len(test.expColumns[i]), len(columns), i)
}
for j, column := range columns {
switch v := test.expColumns[i][j].(type) {
case string:
val := column.GetStringVal()
@ -986,6 +995,171 @@ func TestQuerySQLUnaryWithError(t *testing.T) {
}
}
func TestCRUDIndexes(t *testing.T) {
m := test.RunCommand(t)
defer m.Close()
ctx := context.Background()
gh := server.NewGRPCHandler(m.API)
t.Run("CreateIndex", func(t *testing.T) {
// Try CreateIndex for testindex1
_, err := gh.CreateIndex(ctx, &pb.CreateIndexRequest{Name: "testindex1", Keys: true})
if err != nil {
t.Fatal(err)
}
schema := m.API.Schema(ctx)
if len(schema) != 1 {
t.Fatal("Schema should include one index")
}
if schema[0].Name != "testindex1" {
t.Fatal("Index name not set correctly")
}
if schema[0].Options.Keys != true {
t.Fatal("Index Keys not set correctly")
}
if schema[0].Options.TrackExistence != true {
t.Fatal("Index TrackExistence should be true when created by gRPC")
}
// Try CreateIndex for testindex2
_, err = gh.CreateIndex(ctx, &pb.CreateIndexRequest{Name: "testindex2"})
if err != nil {
t.Fatal(err)
}
schema = m.API.Schema(ctx)
if len(schema) != 2 {
t.Fatal("Schema should include two indexes")
}
_ = m.API.DeleteIndex(ctx, "testindex1")
schema = m.API.Schema(ctx)
if len(schema) != 1 {
t.Fatal("Schema should include one index")
}
if schema[0].Name != "testindex2" {
t.Fatal("Index name not set correctly")
}
if schema[0].Options.Keys != false {
t.Fatal("Index Keys not set correctly")
}
// Check errors for CreateIndex: create index with same name
_, err = gh.CreateIndex(ctx, &pb.CreateIndexRequest{Name: "testindex2"})
errStatus, _ := status.FromError(err)
if errStatus.Code() != codes.AlreadyExists {
t.Fatalf("Error code should be codes.AlreadyExists, but is %v", errStatus.Code())
}
// Check errors for CreateIndex: create index with no name
_, err = gh.CreateIndex(ctx, &pb.CreateIndexRequest{Name: ""})
errStatus, _ = status.FromError(err)
if errStatus.Code() != codes.Unknown {
t.Fatalf("Error code should be codes.Unknown, but is %v", errStatus.Code())
}
// Check errors for CreateIndex: create index with invalid name
_, err = gh.CreateIndex(ctx, &pb.CreateIndexRequest{Name: "💩"})
errStatus, _ = status.FromError(err)
if errStatus.Code() != codes.FailedPrecondition {
t.Fatalf("Error code should be codes.FailedPrecondition, but is %v", errStatus.Code())
}
_ = m.API.DeleteIndex(ctx, "testindex2")
})
t.Run("GetIndex", func(t *testing.T) {
_, err := m.API.CreateIndex(ctx, "testindex1", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
// Check GetIndex for testindex1
resp, err := gh.GetIndex(ctx, &pb.GetIndexRequest{Name: "testindex1"})
if err != nil {
t.Fatal(err)
}
if resp.Index.Name != "testindex1" {
t.Fatalf("Index name does not match: %s", resp.Index.Name)
}
// Check errors for GetIndex: get index that doesn't exist
_, err = gh.GetIndex(ctx, &pb.GetIndexRequest{Name: "wrongname"})
errStatus, _ := status.FromError(err)
if errStatus.Code() != codes.NotFound {
t.Fatalf("Error code should be codes.NotFound, but is %v", errStatus.Code())
}
// Check errors for GetIndex: get index with invalid name
_, err = gh.GetIndex(ctx, &pb.GetIndexRequest{Name: "💩"})
errStatus, _ = status.FromError(err)
if errStatus.Code() != codes.NotFound {
t.Fatalf("Error code should be codes.NotFound, but is %v", errStatus.Code())
}
_ = m.API.DeleteIndex(ctx, "testindex1")
})
t.Run("GetIndexes", func(t *testing.T) {
_, err := m.API.CreateIndex(ctx, "testindex1", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
// Check GetIndexes
resp2, err := gh.GetIndexes(ctx, &pb.GetIndexesRequest{})
if err != nil {
t.Fatal(err)
}
if len(resp2.Indexes) != 1 && resp2.Indexes[0].Name != "testindex1" {
t.Fatalf("GetIndexes did not produce the correct result set: %v", resp2.Indexes)
}
_, err = m.API.CreateIndex(ctx, "testindex2", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
// Check GetIndexes again
resp, err := gh.GetIndexes(ctx, &pb.GetIndexesRequest{})
if err != nil {
t.Fatal(err)
}
if len(resp.Indexes) != 2 {
t.Fatalf("GetIndexes did not produce the correct result set: %v", resp.Indexes)
}
_ = m.API.DeleteIndex(ctx, "testindex1")
_ = m.API.DeleteIndex(ctx, "testindex2")
})
t.Run("DeleteIndexes", func(t *testing.T) {
_, err := m.API.CreateIndex(ctx, "testindex1", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
// Try to delete index
_, err = gh.DeleteIndex(ctx, &pb.DeleteIndexRequest{Name: "testindex1"})
if err != nil {
t.Fatal(err)
}
schema := m.API.Schema(ctx)
if len(schema) != 0 {
t.Fatal("Schema should include no index")
}
// Try to delete non-existing index
_, err = gh.DeleteIndex(ctx, &pb.DeleteIndexRequest{Name: "doesnotexist"})
errStatus, _ := status.FromError(err)
if errStatus.Code() != codes.NotFound {
t.Fatalf("Error code should be codes.NotFound, but is %v", errStatus.Code())
}
})
}
func setUpTestQuerySQLUnary(ctx context.Context, t *testing.T) (gh *server.GRPCHandler, tearDownFunc func()) {
t.Helper()

44
server/pg.go
View file

@ -252,27 +252,38 @@ func pgWriteExtractedTable(w pg.QueryResultWriter, tbl pilosa.ExtractedTable) er
return nil
}
func pgWriteGroupCount(w pg.QueryResultWriter, counts []pilosa.GroupCount) error {
if len(counts) == 0 {
func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error {
groups := counts.Groups()
if len(groups) == 0 {
// Not enough information is available to construct the header.
// This is a significant flaw in the data type.
return nil
}
expectedLen := len(groups[0].Group) + 1
headers := make([]pg.ColumnInfo, len(counts[0].Group)+2)
for i, g := range counts[0].Group {
agg := counts.AggregateColumn()
if agg != "" {
expectedLen++
}
headers := make([]pg.ColumnInfo, expectedLen)
for i, g := range groups[0].Group {
headers[i] = pg.ColumnInfo{
Name: g.Field,
Type: pg.TypeCharoid,
}
}
headers[len(headers)-2] = pg.ColumnInfo{
next := len(groups[0].Group)
headers[next] = pg.ColumnInfo{
Name: "count",
Type: pg.TypeCharoid,
}
headers[len(headers)-1] = pg.ColumnInfo{
Name: "sum",
Type: pg.TypeCharoid,
if agg != "" {
next++
headers[next] = pg.ColumnInfo{
Name: agg,
Type: pg.TypeCharoid,
}
}
err := w.WriteHeader(headers...)
if err != nil {
@ -280,8 +291,10 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts []pilosa.GroupCount) error
}
vals := make([]string, len(headers))
for _, gc := range counts {
for j, g := range gc.Group {
for _, gc := range groups {
var j int
var g pilosa.FieldRow
for j, g = range gc.Group {
var v string
switch {
case g.Value != nil:
@ -293,8 +306,12 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts []pilosa.GroupCount) error
}
vals[j] = v
}
vals[len(vals)-2] = strconv.FormatUint(gc.Count, 10)
vals[len(vals)-1] = strconv.FormatInt(gc.Sum, 10)
j++
vals[j] = strconv.FormatUint(gc.Count, 10)
if agg != "" {
j++
vals[j] = strconv.FormatInt(gc.Agg, 10)
}
err := w.WriteRowText(vals...)
if err != nil {
@ -370,6 +387,9 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error {
case pilosa.ExtractedTable:
return pgWriteExtractedTable(w, result)
case []pilosa.GroupCount:
gc := pilosa.NewGroupCounts("", result...)
return pgWriteGroupCount(w, gc)
case *pilosa.GroupCounts:
return pgWriteGroupCount(w, result)
case pb.ToRowser: // we should avoid protobuf where we can...
return pgWriteRowser(w, result)

5
server/pg_test.go
View file

@ -169,11 +169,10 @@ func TestPostgresHandler(t *testing.T) {
Columns: []pg.ColumnInfo{
{Name: "set", Type: pg.TypeCharoid},
{Name: "count", Type: pg.TypeCharoid},
{Name: "sum", Type: pg.TypeCharoid},
},
Data: [][]string{
{"4", "3", "0"},
{"5", "3", "0"},
{"4", "3"},
{"5", "3"},
},
},
},

2
server/server_test.go
View file

@ -298,7 +298,7 @@ func TestMain_GroupBy(t *testing.T) {
if res, err := m.QueryProtobuf("i", `GroupBy(Rows(generalk), Rows(subk))`); err != nil {
t.Fatal(err)
} else {
test.CheckGroupBy(t, expected, res.Results[0].([]pilosa.GroupCount))
test.CheckGroupBy(t, expected, res.Results[0].(*pilosa.GroupCounts).Groups())
}
}

2
snapshotqueue.go
View file

@ -445,7 +445,7 @@ func (s *prioritySnapshotQueueScanner) ProcessFragment(f *fragment) error {
s.hits++
select {
case s.queue <- snapshotRequest{frag: f, when: time.Now()}:
s.sq.logger.Debugf("found fragment needing snapshot: %s\n", f.path)
s.sq.logger.Debugf("found fragment needing snapshot: %s\n", f.path())
case <-s.ctx.Done():
return io.EOF
}

2
sql/select.go
View file

@ -90,7 +90,7 @@ func (s *SelectHandler) execMappingResult(ctx context.Context, mr *MappingResult
case pproto.ToRowser:
result = res
case []pilosa.GroupCount:
result = pilosa.GroupCounts(res)
result = pilosa.NewGroupCounts("", res...)
case uint64:
result = pproto.ConstRowser{
{

26
test/cluster.go
View file

@ -63,7 +63,7 @@ func (c *Cluster) QueryHTTP(t testing.TB, index, query string) (string, error) {
if len(c.Nodes) == 0 {
t.Fatal("must have at least one node in cluster to query")
}
return c.Nodes[0].Query(t, index, "", query)
}
@ -189,6 +189,30 @@ func (c *Cluster) ImportIntKey(t testing.TB, index, field string, pairs []IntKey
}
}
type IntID struct {
Val int64
ID uint64
}
// ImportIntID imports data into an int field in an unkeyed index.
func (c *Cluster) ImportIntID(t testing.TB, index, field string, pairs []IntID) {
t.Helper()
importRequest := &pilosa.ImportValueRequest{
Index: index,
Field: field,
Shard: math.MaxUint64,
ColumnIDs: make([]uint64, len(pairs)),
Values: make([]int64, len(pairs)),
}
for i, pair := range pairs {
importRequest.Values[i] = pair.Val
importRequest.ColumnIDs[i] = pair.ID
}
if err := c.Nodes[0].API.ImportValue(context.Background(), nil, importRequest); err != nil {
t.Fatalf("importing IntID data: %v", err)
}
}
// KeyID represents a key and an ID for importing data into an index
// and field where one uses string keys and the other does not.
type KeyID struct {