From 121f3fb610e0936f646c3b325b20ba0f8bec647b Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 22 Dec 2020 09:12:41 -0600 Subject: [PATCH 01/13] fix Count(Distinct) bug and add better tests the Distinct call would get precomputed correctly, but then the executeCount would happen in the available shards context of the index. So if the index only had records in (e.g.) shards 10,12,18,22, and all the values of the Distinct call were in shard 0, you'd see 0 results. The fix skips the whole map/reduce step of executeCount (which was basically fake anyway when the argument is precomputed), and just adds up all counts of all the precomputed segments. This currently won't properly count Distinct values from an int field which contains negative numbers... going to add a test and fix for that next. There is also still a key translation bug which is why the one test case is commented out... fix coming for that soon as well. --- executor.go | 22 ++++++++++-- executor_test.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ test/cluster.go | 47 ++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index ae52457c7..52b0a20b7 100644 --- a/executor.go +++ b/executor.go @@ -361,7 +361,7 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr } // handlePreCalls traverses the call tree looking for calls that need -// precomputed values. Right now, that's just Distinct. +// precomputed values (e.g. Distinct, UnionRows, ConstRow...) func (e *executor) handlePreCalls(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) error { if c.Name == "Precomputed" { idx := c.Args["valueidx"].(int64) @@ -4235,9 +4235,27 @@ func (e *executor) executeCount(ctx context.Context, qcx *Qcx, index string, c * return 0, errors.New("Count() only accepts a single bitmap input") } + child := c.Children[0] + + // if the child is precomputed, we'll bypass mapreduce, ignore + // shards, and just count the number of bits + if child.Name == "Precomputed" { + count := uint64(0) + for _, irow := range child.Precomputed { + if row, ok := irow.(*Row); !ok { + return 0, errors.Errorf("unexpected precomputed value type inside count: %+v", irow) + } else { + for _, seg := range row.segments { + count += seg.n + } + } + } + return count, nil + } + // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (_ interface{}, err error) { - row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard) + row, err := e.executeBitmapCallShard(ctx, qcx, index, child, shard) if err != nil { return 0, err } diff --git a/executor_test.go b/executor_test.go index 11a774014..cec51220f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -6742,3 +6742,96 @@ func TestMissingKeyRegression(t *testing.T) { }) } } + +func TestDistinctOnSetsKeyedIndex(t *testing.T) { + c := test.MustRunCluster(t, 3) + defer c.Close() + + // create and populate "likenums" similar to "likes", but no keys on the field + c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums") + c.ImportIDKey(t, "users", "likenums", []test.KeyID{ + {ID: 1, Key: "userA"}, + {ID: 2, Key: "userB"}, + {ID: 3, Key: "userC"}, + {ID: 4, Key: "userD"}, + {ID: 5, Key: "userE"}, + {ID: 6, Key: "userF"}, + {ID: 7, Key: "userA"}, + {ID: 7, Key: "userB"}, + {ID: 7, Key: "userC"}, + {ID: 7, Key: "userD"}, + {ID: 7, Key: "userE"}, + {ID: 7, Key: "userF"}, + }) + + // create and populate "likes" field + c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys()) + c.ImportKeyKey(t, "users", "likes", [][2]string{ + {"molecula", "userA"}, + {"pilosa", "userB"}, + {"pangolin", "userC"}, + {"zebra", "userD"}, + {"toucan", "userE"}, + {"dog", "userF"}, + {"icecream", "userA"}, + {"icecream", "userB"}, + {"icecream", "userC"}, + {"icecream", "userD"}, + {"icecream", "userE"}, + {"icecream", "userF"}, + }) + + tests := []struct { + query string + verifier func(t *testing.T, resp pilosa.QueryResponse) + }{ + { + query: "Count(All())", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if resp.Results[0].(uint64) != 6 { + t.Errorf("expected 6, got %+v", resp.Results[0]) + } + }, + }, + { + query: "Count(Distinct(field=likenums))", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if resp.Results[0].(uint64) != 7 { + t.Errorf("wrong count: %+v", resp.Results[0]) + } + }, + }, + { + query: "Distinct(field=likenums)", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Pos.Columns(), []uint64{1, 2, 3, 4, 5, 6, 7}) { + t.Errorf("wrong values: %+v", resp.Results[0].(pilosa.SignedRow).Pos.Columns()) + } + }, + }, + { + query: "Count(Distinct(field=likes))", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if resp.Results[0].(uint64) != 7 { + t.Errorf("wrong count: %+v", resp.Results[0]) + } + }, + }, + // { + // query: "Distinct(field=likes)", + // verifier: func(t *testing.T, resp pilosa.QueryResponse) { + // if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pilosa", "pangolin", "zebra", "toucan", "dog", "icecream"}) { + // t.Errorf("wrong values: %+v", resp.Results[0]) + // } + // }, + // }, + } + + for i, tst := range tests { + t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) { + resp := c.Query(t, "users", tst.query) + fmt.Println(resp.Results[0]) + tst.verifier(t, resp) + }) + } +} diff --git a/test/cluster.go b/test/cluster.go index 5dfb4edfb..0a5d2c451 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -108,6 +108,53 @@ func (c *Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uin } } +// ImportKeyKey imports data into an index where both the index and +// the field are using string keys. +func (c *Cluster) ImportKeyKey(t testing.TB, index, field string, valAndRecKeys [][2]string) { + t.Helper() + importRequest := &pilosa.ImportRequest{ + Index: index, + Field: field, + RowKeys: make([]string, len(valAndRecKeys)), + ColumnKeys: make([]string, len(valAndRecKeys)), + } + for i, vk := range valAndRecKeys { + importRequest.RowKeys[i] = vk[0] + importRequest.ColumnKeys[i] = vk[1] + } + err := c.Nodes[0].API.Import(context.Background(), nil, importRequest) + if err != nil { + t.Fatalf("importing keykey data: %v", err) + } +} + +// KeyID represents a key and an ID for importing data into an index +// and field where one uses string keys and the other does not. +type KeyID struct { + Key string + ID uint64 +} + +// ImportIDKey imports data into an index where the index is using +// keys, but the field is not. +func (c *Cluster) ImportIDKey(t testing.TB, index, field string, pairs []KeyID) { + t.Helper() + importRequest := &pilosa.ImportRequest{ + Index: index, + Field: field, + RowIDs: make([]uint64, len(pairs)), + ColumnKeys: make([]string, len(pairs)), + } + for i, pair := range pairs { + importRequest.RowIDs[i] = pair.ID + importRequest.ColumnKeys[i] = pair.Key + } + err := c.Nodes[0].API.Import(context.Background(), nil, importRequest) + if err != nil { + t.Fatalf("importing IDKey data: %v", err) + } +} + // CreateField creates the index (if necessary) and field specified. func (c *Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field { t.Helper() From 385381e5f3f827c53f2705be80cf6247fc231581 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 22 Dec 2020 11:42:57 -0600 Subject: [PATCH 02/13] fix issue where a shard with no data can cause query to fail add Distinct test with integer data, and because one of the records had a null value (and was in a shard by itself), it uncovered this issue. I added a special error type if a view or fragment is not found when so that we can match against it and ignore it when calculating the results for a query. I also added an implementation within executeCount to handle the SignedRow case, but discovered that handlePrecalls always dumps the negative data and that will be a bigger thing to fix --- executor.go | 20 ++++++++++++++++---- executor_test.go | 40 ++++++++++++++++++++++++++++++++++++++-- rrtx.go | 7 +++++-- test/cluster.go | 26 ++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 8 deletions(-) diff --git a/executor.go b/executor.go index 52b0a20b7..1668fa218 100644 --- a/executor.go +++ b/executor.go @@ -1516,7 +1516,10 @@ func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldNam existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*0, ShardWidth*1) if err != nil { - return result, err + if _, ok := errors.Cause(err).(ViewOrFragmentNotFound); ok { + return result, nil + } + return result, errors.Wrap(err, "getting exists bitmap") } if filterBitmap != nil { existsBitmap = existsBitmap.Intersect(filterBitmap) @@ -1527,6 +1530,7 @@ func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldNam signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*1, ShardWidth*2) if err != nil { + // TODO wtf... if there's any error getting the sign bitmap we just return an empty result and move on? return result, nil } @@ -4242,12 +4246,20 @@ func (e *executor) executeCount(ctx context.Context, qcx *Qcx, index string, c * if child.Name == "Precomputed" { count := uint64(0) for _, irow := range child.Precomputed { - if row, ok := irow.(*Row); !ok { - return 0, errors.Errorf("unexpected precomputed value type inside count: %+v", irow) - } else { + switch row := irow.(type) { + case *Row: for _, seg := range row.segments { count += seg.n } + case SignedRow: + for _, seg := range row.Pos.segments { + count += seg.n + } + for _, seg := range row.Neg.segments { + count += seg.n + } + default: + return 0, errors.Errorf("unexpected precomputed value type inside count: %+v", row) } } return count, nil diff --git a/executor_test.go b/executor_test.go index cec51220f..dd9363bf7 100644 --- a/executor_test.go +++ b/executor_test.go @@ -6747,7 +6747,7 @@ func TestDistinctOnSetsKeyedIndex(t *testing.T) { c := test.MustRunCluster(t, 3) defer c.Close() - // create and populate "likenums" similar to "likes", but no keys on the field + // Create and populate "likenums" similar to "likes", but without keys on the field. c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums") c.ImportIDKey(t, "users", "likenums", []test.KeyID{ {ID: 1, Key: "userA"}, @@ -6764,7 +6764,7 @@ func TestDistinctOnSetsKeyedIndex(t *testing.T) { {ID: 7, Key: "userF"}, }) - // create and populate "likes" field + // Create and populate "likes" field. c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys()) c.ImportKeyKey(t, "users", "likes", [][2]string{ {"molecula", "userA"}, @@ -6781,6 +6781,16 @@ func TestDistinctOnSetsKeyedIndex(t *testing.T) { {"icecream", "userF"}, }) + // Create and populate "affinity" int field with negative, positive, zero and null values. + c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "affinity", pilosa.OptFieldTypeInt(-1000, 1000)) + c.ImportIntKey(t, "users", "affinity", []test.IntKey{ + {Val: 10, Key: "userA"}, + {Val: -10, Key: "userB"}, + {Val: 5, Key: "userC"}, + {Val: -5, Key: "userD"}, + {Val: 0, Key: "userE"}, + }) + tests := []struct { query string verifier func(t *testing.T, resp pilosa.QueryResponse) @@ -6817,6 +6827,32 @@ func TestDistinctOnSetsKeyedIndex(t *testing.T) { } }, }, + { + query: "Distinct(field=affinity)", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Pos.Columns(), []uint64{0, 5, 10}) { + t.Errorf("wrong positive records: %+v", resp.Results[0].(pilosa.SignedRow).Pos.Columns()) + } + if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Neg.Columns(), []uint64{5, 10}) { + t.Errorf("wrong negative records: %+v", resp.Results[0].(pilosa.SignedRow).Neg.Columns()) + } + }, + }, + // handling this case properly will require changing the way + // that precomputed data is stored on Call objects. Currently + // if a Distinct is at all nested (e.g. within a Count) it + // gets handled by executor.handlePreCalls which assumes that + // only the positive values are worthwhile. + // { + // query: "Count(Distinct(field=affinity))", + // verifier: func(t *testing.T, resp pilosa.QueryResponse) { + // if resp.Results[0].(uint64) != 5 { + // t.Errorf("wrong number of values: %+v", resp.Results[0]) + // } + // }, + // }, + + // this case doesn't work due to the missing index issue // { // query: "Distinct(field=likes)", // verifier: func(t *testing.T, resp pilosa.QueryResponse) { diff --git a/rrtx.go b/rrtx.go index 0cfb5fba9..0b4d7d1e7 100644 --- a/rrtx.go +++ b/rrtx.go @@ -30,6 +30,7 @@ import ( rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg" "github.com/pilosa/pilosa/v2/roaring" txkey "github.com/pilosa/pilosa/v2/short_txkey" + //txkey "github.com/pilosa/pilosa/v2/txkey" "github.com/pkg/errors" ) @@ -372,13 +373,13 @@ func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*frag v := f.view(view) if v == nil { - return nil, errors.Errorf("view not found: %q", view) + return nil, ViewOrFragmentNotFound(errors.Errorf("view not found: %q", view)) } frag := v.Fragment(shard) if frag == nil { - return nil, fmt.Errorf("fragment not found: %q / %q / %d", field, view, shard) + return nil, ViewOrFragmentNotFound(errors.Errorf("fragment not found: %q / %q / %d", field, view, shard)) } // Note: we cannot cache frag into tx.fragment. @@ -388,6 +389,8 @@ func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*frag return frag, nil } +type ViewOrFragmentNotFound error + func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { frag, err := tx.getFragment(index, field, view, shard) if err != nil { diff --git a/test/cluster.go b/test/cluster.go index 0a5d2c451..74b1e31ff 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "io/ioutil" + "math" "path" "strconv" "strings" @@ -128,6 +129,31 @@ func (c *Cluster) ImportKeyKey(t testing.TB, index, field string, valAndRecKeys } } +// IntKey is a string key and a signed integer value. +type IntKey struct { + Val int64 + Key string +} + +// ImportIntKey imports int data into an index which uses string keys. +func (c *Cluster) ImportIntKey(t testing.TB, index, field string, pairs []IntKey) { + t.Helper() + importRequest := &pilosa.ImportValueRequest{ + Index: index, + Field: field, + Shard: math.MaxUint64, + ColumnKeys: make([]string, len(pairs)), + Values: make([]int64, len(pairs)), + } + for i, pair := range pairs { + importRequest.Values[i] = pair.Val + importRequest.ColumnKeys[i] = pair.Key + } + if err := c.Nodes[0].API.ImportValue(context.Background(), nil, importRequest); err != nil { + t.Fatalf("importing IntKey data: %v", err) + } +} + // KeyID represents a key and an ID for importing data into an index // and field where one uses string keys and the other does not. type KeyID struct { From 9ee5f52a11b77359e4cc0db1e4239727556837e4 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 23 Dec 2020 14:14:51 -0600 Subject: [PATCH 03/13] fix some Distinct key translation issues (e.g. empty index) This commit changes executeDistinct to return either a *Row or a SignedRow (instead of only being able to return a SignedRow). Distinct on a set field will return a *Row while an int field will still return a signed row. We then add Field and Index fields to the Row object so that we can determine how to translate the rows IDs to keys (if needed). This adds a lot of logic around the translation which fixes bugs where Distinct calls would fail to get translated. There are, I think, still issues if you were to try to join a keyed field to a keyed index which wasn't explicitly specified as the field's foreign index. The IDs in the field wouldn't be using the same translation as the IDs in the index, so the query might appear to work but give incorrect results. --- encoding/proto/proto.go | 4 + executor.go | 184 +++++++++++++++++++---- executor_test.go | 43 ++++-- internal/public.pb.go | 314 ++++++++++++++++++++++++++-------------- internal/public.proto | 2 + row.go | 11 +- 6 files changed, 408 insertions(+), 150 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 1631e47dd..ec3e3c950 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -1453,6 +1453,8 @@ func (s Serializer) decodeRow(pr *internal.Row) *pilosa.Row { } r.Attrs = s.decodeAttrs(pr.Attrs) r.Keys = pr.Keys + r.Index = pr.Index + r.Field = pr.Field return r } @@ -1700,6 +1702,8 @@ func (s Serializer) encodeRow(r *pilosa.Row) *internal.Row { ir := &internal.Row{ Keys: r.Keys, Attrs: s.encodeAttrs(r.Attrs), + Index: r.Index, + Field: r.Field, } if s.RoaringRows { ir.Roaring = r.Roaring() diff --git a/executor.go b/executor.go index 1668fa218..4a0fa5d27 100644 --- a/executor.go +++ b/executor.go @@ -563,6 +563,7 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q } else { v, err = e.executeCall(ctx, qcx, index, call, shards, opt) } + if err != nil { return nil, err } @@ -1080,8 +1081,9 @@ func (e *executor) executeSum(ctx context.Context, qcx *Qcx, index string, c *pq return other, nil } -// executeDistinct executes a Distinct call on a field. -func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (SignedRow, error) { +// executeDistinct executes a Distinct call on a field. It returns a +// SignedRow for int fields and a *Row for set/mutex/time fields. +func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinct") defer span.Finish() @@ -1099,21 +1101,31 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, // Merge returned results at coordinating node. reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { - other, _ := prev.(SignedRow) if err := ctx.Err(); err != nil { return err } - return other.union(v.(SignedRow)) + switch other := prev.(type) { + case SignedRow: + return other.union(v.(SignedRow)) + case *Row: + return other.Union(v.(*Row)) + case nil: + return v + default: + return errors.Errorf("unexpected return type from executeDistinctShard: %+v %T", other, other) + } } result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return SignedRow{}, err + return nil, err } - other, _ := result.(SignedRow) - other.field = field - return other, nil + if other, ok := result.(SignedRow); ok { + other.field = field + } + + return result, nil } // executeMin executes a Min() call. @@ -1302,7 +1314,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, qcx *Qcx, index string reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(*Row) if other == nil { - + // TODO... what's going on on the following line other = NewRow() // bug! this row ends up containing Badger Txn data that should be accessed outside the Txn. } if err := ctx.Err(); err != nil { @@ -1398,14 +1410,20 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, qcx *Qcx, index s // executeDistinctShard executes a Distinct call on a single shard, yielding // a SignedRow of the values found. -func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index string, fieldName string, c *pql.Call, shard uint64) (result SignedRow, err error) { +func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index string, fieldName string, c *pql.Call, shard uint64) (result interface{}, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinctShard") defer span.Finish() idx := e.Holder.Index(index) field := e.Holder.Field(index, fieldName) if field == nil { - return SignedRow{}, ErrFieldNotFound + return nil, ErrFieldNotFound + } + bsig := field.bsiGroup(fieldName) + if bsig == nil { + result = new(Row) + } else { + result = SignedRow{} } var filter *Row @@ -1427,28 +1445,27 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str // can go ahead and save time by returning the empty results, // because the filter excluded everything. if filterBitmap == nil || !filterBitmap.Any() { - return SignedRow{}, nil + return result, nil } } - bsig := field.bsiGroup(fieldName) if bsig == nil { return executeDistinctShardSet(ctx, qcx, idx, fieldName, shard, filterBitmap) } return executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) } -func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result SignedRow, err0 error) { +func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result *Row, err0 error) { index := idx.Name() tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) if err != nil { - return SignedRow{}, err + return nil, err } defer finisher(&err0) fragData, _, err := tx.ContainerIterator(index, fieldName, "standard", shard, 0) if err != nil { - return SignedRow{}, errors.Wrap(err, "getting fragment data") + return nil, errors.Wrap(err, "getting fragment data") } defer fragData.Close() // We can't grab the containers "for each row" from the set-type field, @@ -1486,20 +1503,22 @@ func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldNam if roaring.IntersectionAny(c, filter[k%(1<=0), field=affinity)", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Pos.Columns(), []uint64{0, 5, 10}) { + t.Errorf("wrong positive records: %+v", resp.Results[0].(pilosa.SignedRow).Pos.Columns()) + } + if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Neg.Columns(), []uint64{}) { + t.Errorf("wrong negative records: %+v", resp.Results[0].(pilosa.SignedRow).Neg.Columns()) + } + }, + }, + { + query: "Count(Distinct(Row(affinity>=0), field=affinity))", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if resp.Results[0].(uint64) != 3 { + t.Errorf("wrong number of values: %+v", resp.Results[0]) + } + }, + }, + // handling this case properly will require changing the way // that precomputed data is stored on Call objects. Currently // if a Distinct is at all nested (e.g. within a Count) it @@ -6851,22 +6871,19 @@ func TestDistinctOnSetsKeyedIndex(t *testing.T) { // } // }, // }, - - // this case doesn't work due to the missing index issue - // { - // query: "Distinct(field=likes)", - // verifier: func(t *testing.T, resp pilosa.QueryResponse) { - // if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pilosa", "pangolin", "zebra", "toucan", "dog", "icecream"}) { - // t.Errorf("wrong values: %+v", resp.Results[0]) - // } - // }, - // }, + { + query: "Distinct(field=likes)", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pilosa", "pangolin", "zebra", "toucan", "dog", "icecream"}) { + t.Errorf("wrong values: %+v", resp.Results[0]) + } + }, + }, } for i, tst := range tests { t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) { resp := c.Query(t, "users", tst.query) - fmt.Println(resp.Results[0]) tst.verifier(t, resp) }) } diff --git a/internal/public.pb.go b/internal/public.pb.go index b3038896d..b334a48e3 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -28,6 +28,8 @@ type Row struct { Keys []string `protobuf:"bytes,3,rep,name=Keys,proto3" json:"Keys,omitempty"` Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,proto3" json:"Attrs,omitempty"` Roaring []byte `protobuf:"bytes,4,opt,name=Roaring,proto3" json:"Roaring,omitempty"` + Index string `protobuf:"bytes,5,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,6,opt,name=Field,proto3" json:"Field,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -94,6 +96,20 @@ func (m *Row) GetRoaring() []byte { return nil } +func (m *Row) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *Row) GetField() string { + if m != nil { + return m.Field + } + return "" +} + type RowMatrix struct { Rows []*Row `protobuf:"bytes,1,rep,name=Rows,proto3" json:"Rows,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -2667,113 +2683,113 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1682 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0xdb, 0xca, - 0x11, 0x37, 0x45, 0xca, 0x92, 0x46, 0xb2, 0xe3, 0xb7, 0xd1, 0x7b, 0x25, 0x52, 0xc7, 0x4f, 0x25, - 0xdc, 0x3e, 0xb5, 0x28, 0x1c, 0x38, 0x4d, 0x82, 0x5c, 0xda, 0xc6, 0x8e, 0x9c, 0x9a, 0x48, 0xed, - 0xa6, 0x2b, 0xc3, 0xb9, 0x15, 0xa0, 0xa5, 0xad, 0x43, 0x94, 0x12, 0x55, 0x8a, 0x8a, 0xec, 0x4b, - 0x81, 0x7e, 0x86, 0x5c, 0xfa, 0x11, 0x7a, 0xea, 0x87, 0xe8, 0xa5, 0x3d, 0xf6, 0x58, 0xa0, 0x97, - 0x22, 0xed, 0xb7, 0xe8, 0xa5, 0x98, 0x59, 0x2e, 0x77, 0x49, 0xd1, 0x8e, 0x11, 0xbc, 0xdb, 0xce, - 0x9f, 0x9d, 0x9d, 0xf9, 0xcd, 0xec, 0xec, 0x90, 0xd0, 0x99, 0x2d, 0x2e, 0xa2, 0x70, 0xb4, 0x37, - 0x4b, 0xe2, 0x34, 0x66, 0xcd, 0x70, 0x9a, 0x8a, 0x64, 0x1a, 0x44, 0xde, 0x1c, 0x6c, 0x1e, 0x2f, - 0x99, 0x0b, 0x8d, 0x97, 0x71, 0xb4, 0x98, 0x4c, 0xe7, 0xae, 0xd5, 0xb3, 0xfb, 0x0e, 0x57, 0x24, - 0x63, 0xe0, 0xbc, 0x16, 0xd7, 0x73, 0xd7, 0xee, 0xd9, 0xfd, 0x16, 0xa7, 0x35, 0xdb, 0x85, 0xfa, - 0x41, 0x9a, 0x26, 0x73, 0xb7, 0xd6, 0xb3, 0xfb, 0xed, 0xc7, 0x9b, 0x7b, 0xca, 0xdc, 0x1e, 0xb2, - 0xb9, 0x14, 0xa2, 0x4d, 0x1e, 0x07, 0x49, 0x38, 0xbd, 0x74, 0x9d, 0x9e, 0xd5, 0xef, 0x70, 0x45, - 0x7a, 0x7b, 0xd0, 0xe2, 0xf1, 0xf2, 0x24, 0x48, 0x93, 0xf0, 0x8a, 0x7d, 0x0f, 0x1c, 0x1e, 0x2f, - 0xe5, 0xb9, 0xed, 0xc7, 0x1b, 0xda, 0x16, 0x8f, 0x97, 0x9c, 0x44, 0xde, 0x09, 0xb4, 0x86, 0xe1, - 0xe5, 0x54, 0x8c, 0xd1, 0xd5, 0xaf, 0xc1, 0x7e, 0x13, 0xa3, 0xba, 0xb5, 0xaa, 0x8e, 0x12, 0x54, - 0x38, 0x15, 0x97, 0x6e, 0xad, 0x52, 0xe1, 0x54, 0x5c, 0x7a, 0xcf, 0x61, 0x93, 0xc7, 0x4b, 0x7f, - 0x2c, 0xa6, 0x69, 0xf8, 0xdb, 0x50, 0x24, 0x14, 0x64, 0xee, 0x83, 0x23, 0x0f, 0xcd, 0x03, 0xaf, - 0xe9, 0xc0, 0xbd, 0x07, 0xb0, 0xee, 0x0f, 0x7e, 0x19, 0xce, 0x53, 0xb6, 0x05, 0xb6, 0x3f, 0x50, - 0x1b, 0x70, 0xe9, 0xf9, 0xf0, 0xc5, 0xd1, 0x55, 0x9a, 0x04, 0xa3, 0x54, 0x8c, 0xfd, 0x81, 0x84, - 0x8f, 0x6d, 0x42, 0xcd, 0x1f, 0x90, 0xaf, 0x0e, 0xaf, 0xf9, 0x03, 0xb6, 0x0b, 0xce, 0x79, 0x10, - 0x29, 0xe0, 0xb6, 0xb4, 0x73, 0xd2, 0x2c, 0x27, 0xa9, 0x77, 0x51, 0x30, 0x95, 0xe1, 0xf4, 0x15, - 0xac, 0xbf, 0x0a, 0x45, 0x34, 0x96, 0x87, 0xb6, 0x78, 0x46, 0xb1, 0xa7, 0x3a, 0x75, 0xd2, 0xea, - 0x77, 0xb5, 0xd5, 0x15, 0x87, 0xf2, 0xbc, 0x7a, 0x0f, 0xa1, 0xf1, 0x5a, 0x5c, 0x53, 0x2c, 0x2a, - 0x52, 0xcb, 0x88, 0xf4, 0x5f, 0x16, 0xdc, 0xcf, 0x77, 0x9f, 0x05, 0x17, 0x91, 0x38, 0x0f, 0xa2, - 0x85, 0x60, 0xbb, 0x2a, 0x6e, 0xab, 0xca, 0xff, 0xe3, 0x35, 0xc2, 0x82, 0x7d, 0x93, 0x63, 0x87, - 0x6a, 0x5f, 0x68, 0xb5, 0xec, 0xc8, 0xe3, 0xb5, 0xac, 0x92, 0xb6, 0xa1, 0x79, 0x38, 0xf4, 0xc9, - 0xb4, 0x6b, 0xf7, 0xac, 0xbe, 0x7d, 0xbc, 0xc6, 0x73, 0x0e, 0x7b, 0x00, 0x8d, 0x93, 0x45, 0x2a, - 0xae, 0xfc, 0x01, 0x55, 0x90, 0x73, 0xbc, 0xc6, 0x15, 0x03, 0x77, 0xd2, 0xf2, 0xb5, 0xb8, 0x76, - 0xeb, 0x3d, 0xab, 0xdf, 0xc2, 0x9d, 0x8a, 0xc3, 0xba, 0xe0, 0x1c, 0xc6, 0x71, 0xe4, 0xae, 0xf7, - 0xac, 0x7e, 0x13, 0x4f, 0x43, 0xea, 0xb0, 0x01, 0x75, 0x32, 0xec, 0xfd, 0x01, 0xba, 0xc5, 0xe0, - 0xb2, 0x74, 0x31, 0xb0, 0xd1, 0x9e, 0x95, 0xd9, 0x43, 0x82, 0x6d, 0x51, 0x0a, 0x6b, 0xd9, 0xf9, - 0x98, 0xc4, 0xa7, 0xb0, 0x4e, 0x66, 0xe4, 0xa5, 0x68, 0x3f, 0x7e, 0x58, 0x01, 0xb8, 0x86, 0x8c, - 0x67, 0xca, 0x87, 0x2d, 0x42, 0xfc, 0x57, 0x89, 0x3f, 0xf0, 0x7e, 0x5a, 0x06, 0x97, 0x72, 0x89, - 0x89, 0x38, 0x0d, 0x26, 0x42, 0x9e, 0xcf, 0x69, 0x8d, 0xbc, 0xb3, 0xeb, 0x99, 0x20, 0x07, 0x5a, - 0x9c, 0xd6, 0xde, 0x1f, 0x2d, 0xd8, 0x2c, 0xee, 0x47, 0x9f, 0x8c, 0xea, 0xb8, 0xc5, 0x27, 0xd2, - 0xca, 0x8b, 0xe7, 0x79, 0xb9, 0x78, 0x76, 0x6e, 0xda, 0x57, 0xae, 0x9f, 0x9f, 0x81, 0xf3, 0x26, - 0x08, 0x93, 0x95, 0x0a, 0xdf, 0x92, 0x10, 0xda, 0xe4, 0xae, 0x2d, 0x73, 0x51, 0x7f, 0x19, 0x2f, - 0xa6, 0xa9, 0xc4, 0x90, 0x4b, 0xc2, 0x3b, 0x82, 0x16, 0xee, 0x97, 0x81, 0x7b, 0xd2, 0x58, 0x56, - 0x56, 0x46, 0x3f, 0x41, 0x2e, 0x97, 0x07, 0x75, 0xa1, 0x4e, 0xca, 0x19, 0x12, 0x92, 0xf0, 0x8e, + // 1694 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x73, 0x1b, 0x4b, + 0x11, 0xf7, 0x6a, 0x57, 0x96, 0xd4, 0x92, 0x1d, 0xbf, 0x89, 0xde, 0x63, 0x2b, 0x38, 0x7e, 0x62, + 0xcb, 0xf0, 0x04, 0x45, 0x39, 0xe5, 0x90, 0xa4, 0x72, 0x01, 0x62, 0x47, 0x0e, 0xde, 0x0a, 0x36, + 0x61, 0xe4, 0x72, 0x6e, 0x54, 0xad, 0xa5, 0xc1, 0xd9, 0x62, 0xa5, 0x15, 0xab, 0x55, 0x64, 0x5f, + 0xa8, 0xe2, 0x33, 0xe4, 0xc2, 0x8d, 0x2b, 0x27, 0x3e, 0x04, 0x17, 0x38, 0x72, 0xa4, 0x8a, 0x0b, + 0x15, 0xf8, 0x16, 0x5c, 0xa8, 0xee, 0xd9, 0xd9, 0x99, 0x5d, 0xad, 0x1d, 0x57, 0x8a, 0xdb, 0xf4, + 0x9f, 0xe9, 0xe9, 0xfe, 0x75, 0x4f, 0x4f, 0xef, 0x42, 0x67, 0xb6, 0xb8, 0x88, 0xc2, 0xd1, 0xde, + 0x2c, 0x89, 0xd3, 0x98, 0x35, 0xc3, 0x69, 0x2a, 0x92, 0x69, 0x10, 0x79, 0x7f, 0xb4, 0xc0, 0xe6, + 0xf1, 0x92, 0xb9, 0xd0, 0x78, 0x19, 0x47, 0x8b, 0xc9, 0x74, 0xee, 0x5a, 0x3d, 0xbb, 0xef, 0x70, + 0x45, 0x32, 0x06, 0xce, 0x6b, 0x71, 0x3d, 0x77, 0xed, 0x9e, 0xdd, 0x6f, 0x71, 0x5a, 0xb3, 0x5d, + 0xa8, 0x1f, 0xa4, 0x69, 0x32, 0x77, 0x6b, 0x3d, 0xbb, 0xdf, 0x7e, 0xbc, 0xb9, 0xa7, 0xec, 0xed, + 0x21, 0x9b, 0x4b, 0x21, 0xda, 0xe4, 0x71, 0x90, 0x84, 0xd3, 0x4b, 0xd7, 0xe9, 0x59, 0xfd, 0x0e, + 0x57, 0x24, 0xeb, 0x42, 0xdd, 0x9f, 0x8e, 0xc5, 0x95, 0x5b, 0xef, 0x59, 0xfd, 0x16, 0x97, 0x04, + 0x72, 0x5f, 0x85, 0x22, 0x1a, 0xbb, 0xeb, 0x92, 0x4b, 0x84, 0xb7, 0x07, 0x2d, 0x1e, 0x2f, 0x4f, + 0x82, 0x34, 0x09, 0xaf, 0xd8, 0x77, 0xc0, 0xe1, 0xf1, 0x52, 0xfa, 0xd8, 0x7e, 0xbc, 0xa1, 0xcf, + 0xe5, 0xf1, 0x92, 0x93, 0xc8, 0x3b, 0x81, 0xd6, 0x30, 0xbc, 0x9c, 0x8a, 0x31, 0x86, 0xf5, 0x35, + 0xd8, 0x6f, 0x62, 0x54, 0xb7, 0x56, 0xd5, 0x51, 0x82, 0x0a, 0xa7, 0xe2, 0xd2, 0xad, 0x55, 0x2a, + 0x9c, 0x8a, 0x4b, 0xef, 0x39, 0x6c, 0xf2, 0x78, 0xe9, 0x8f, 0xc5, 0x34, 0x0d, 0x7f, 0x1d, 0x8a, + 0x84, 0x00, 0xc9, 0x7d, 0x70, 0xe4, 0xa1, 0x39, 0x48, 0x35, 0x0d, 0x92, 0xf7, 0x00, 0xd6, 0xfd, + 0xc1, 0xcf, 0xc3, 0x79, 0xca, 0xb6, 0xc0, 0xf6, 0x07, 0x6a, 0x03, 0x2e, 0x3d, 0x1f, 0xbe, 0x38, + 0xba, 0x4a, 0x93, 0x60, 0x94, 0x8a, 0xb1, 0x3f, 0x90, 0x50, 0xb3, 0x4d, 0xa8, 0xf9, 0x03, 0xf2, + 0xd5, 0xe1, 0x35, 0x7f, 0xc0, 0x76, 0xc1, 0x39, 0x0f, 0x22, 0x05, 0xf2, 0x96, 0x76, 0x4e, 0x9a, + 0xe5, 0x24, 0xf5, 0x2e, 0x0a, 0xa6, 0x32, 0x9c, 0xbe, 0x82, 0x75, 0x42, 0x4f, 0x1e, 0xda, 0xe2, + 0x19, 0xc5, 0x9e, 0xea, 0x34, 0x4b, 0xab, 0xdf, 0xd6, 0x56, 0x57, 0x1c, 0xca, 0x6b, 0xc0, 0x7b, + 0x08, 0x8d, 0xd7, 0xe2, 0x9a, 0x62, 0x51, 0x91, 0x5a, 0x46, 0xa4, 0xff, 0xb4, 0xe0, 0x7e, 0xbe, + 0xfb, 0x2c, 0xb8, 0x88, 0xc4, 0x79, 0x10, 0x2d, 0x04, 0xdb, 0x55, 0x71, 0x5b, 0x55, 0xfe, 0x1f, + 0xaf, 0x11, 0x16, 0xec, 0x9b, 0x1c, 0x3b, 0x54, 0xfb, 0x42, 0xab, 0x65, 0x47, 0x1e, 0xaf, 0x65, + 0x55, 0xb7, 0x0d, 0xcd, 0xc3, 0xa1, 0x4f, 0xa6, 0x5d, 0xbb, 0x67, 0xf5, 0xed, 0xe3, 0x35, 0x9e, + 0x73, 0xd8, 0x03, 0x68, 0x9c, 0x2c, 0x52, 0x71, 0xe5, 0x0f, 0xa8, 0xda, 0x9c, 0xe3, 0x35, 0xae, + 0x18, 0xb8, 0x93, 0x96, 0xaf, 0xc5, 0xb5, 0x2c, 0x39, 0xdc, 0xa9, 0x38, 0xac, 0x0b, 0xce, 0x61, + 0x1c, 0x47, 0x54, 0x76, 0x4d, 0x3c, 0x0d, 0xa9, 0xc3, 0x06, 0xd4, 0xc9, 0xb0, 0xf7, 0x3b, 0xe8, + 0x16, 0x83, 0xcb, 0xd2, 0xc5, 0xc0, 0x46, 0x7b, 0x56, 0x66, 0x0f, 0x09, 0xb6, 0x45, 0x29, 0xac, + 0x65, 0xe7, 0x63, 0x12, 0x9f, 0xc2, 0x3a, 0x99, 0x91, 0x17, 0xa8, 0xfd, 0xf8, 0x61, 0x05, 0xe0, + 0x1a, 0x32, 0x9e, 0x29, 0x1f, 0xb6, 0x08, 0xf1, 0x5f, 0x24, 0xfe, 0xc0, 0xfb, 0x71, 0x19, 0x5c, + 0xca, 0x25, 0x26, 0xe2, 0x34, 0x98, 0x08, 0x79, 0x3e, 0xa7, 0x35, 0xf2, 0xce, 0xae, 0x67, 0x82, + 0x1c, 0x68, 0x71, 0x5a, 0x7b, 0xbf, 0xb7, 0x60, 0xb3, 0xb8, 0x1f, 0x7d, 0x32, 0xaa, 0xe3, 0x16, + 0x9f, 0x48, 0x2b, 0x2f, 0x9e, 0xe7, 0xe5, 0xe2, 0xd9, 0xb9, 0x69, 0x5f, 0xb9, 0x7e, 0x7e, 0x02, + 0xce, 0x9b, 0x20, 0x4c, 0x56, 0x2a, 0x7c, 0x4b, 0x42, 0x68, 0x93, 0xbb, 0xb6, 0xcc, 0x45, 0xfd, + 0x65, 0xbc, 0x98, 0xa6, 0x12, 0x43, 0x2e, 0x09, 0xef, 0x08, 0x5a, 0xb8, 0x5f, 0x06, 0xee, 0x49, + 0x63, 0x59, 0x59, 0x19, 0xbd, 0x07, 0xb9, 0x5c, 0x1e, 0x94, 0xb7, 0x92, 0x9a, 0xd9, 0x4a, 0x8e, 0x01, 0x50, 0x3a, 0x97, 0x76, 0x76, 0xa1, 0x4e, 0x54, 0x06, 0x42, 0xd9, 0x90, 0x14, 0xde, 0x60, - 0xe9, 0x21, 0xd4, 0xfd, 0x69, 0xfa, 0xec, 0x09, 0x8a, 0x65, 0x41, 0xa2, 0x37, 0x36, 0xcf, 0x4a, - 0x66, 0x01, 0x4d, 0x09, 0x5d, 0xbc, 0xd4, 0x06, 0x2c, 0xc3, 0x00, 0x72, 0xb1, 0xad, 0x0c, 0x54, - 0x9c, 0x44, 0xe0, 0xb5, 0xe5, 0xf1, 0x52, 0x43, 0x92, 0x51, 0xec, 0xfb, 0xea, 0x14, 0x87, 0x62, - 0xbe, 0x67, 0x5c, 0x25, 0xf4, 0x42, 0x1d, 0xfb, 0x1b, 0x80, 0x5f, 0x24, 0xf1, 0x62, 0x46, 0xa0, - 0xb1, 0x3e, 0xd4, 0x89, 0xca, 0xe2, 0x63, 0x7a, 0x93, 0xf2, 0x8d, 0x4b, 0x85, 0x6a, 0xd0, 0x31, - 0x39, 0xc3, 0xc5, 0x44, 0xde, 0x34, 0x8e, 0x4b, 0x2c, 0xa5, 0xe6, 0x79, 0x10, 0xe5, 0xe2, 0xf3, - 0x20, 0xca, 0xe2, 0xc6, 0x65, 0xd1, 0x8c, 0xad, 0xcc, 0x3c, 0x80, 0xe6, 0xab, 0x28, 0x0e, 0x52, - 0x54, 0x46, 0x5b, 0x16, 0xcf, 0x69, 0xb6, 0x0f, 0x30, 0x10, 0xa3, 0x70, 0x12, 0x44, 0x28, 0x75, - 0xca, 0x0d, 0x20, 0x93, 0x71, 0x43, 0xc9, 0x7b, 0x0a, 0x8d, 0x8c, 0xaa, 0xc6, 0x1e, 0xb9, 0xc3, - 0x51, 0x10, 0x09, 0xe5, 0x05, 0x11, 0xde, 0x5b, 0xd8, 0x90, 0xc5, 0x88, 0xcf, 0xcd, 0x50, 0xa4, - 0x77, 0x28, 0xc5, 0x3b, 0x3d, 0x5c, 0xde, 0x9f, 0x2d, 0x70, 0x70, 0xa5, 0x0c, 0x58, 0xda, 0x80, - 0x79, 0x1b, 0x1d, 0x79, 0x1b, 0x59, 0x0f, 0xda, 0xc3, 0x14, 0xdf, 0x35, 0xdd, 0xc6, 0x5a, 0xdc, - 0x64, 0x21, 0x5e, 0xfe, 0x34, 0xd5, 0xe9, 0xb6, 0x79, 0x4e, 0xb3, 0x6d, 0x68, 0x61, 0x6f, 0x92, - 0x42, 0x6c, 0x64, 0x4d, 0xae, 0x19, 0x6c, 0x07, 0x40, 0x21, 0xbb, 0x10, 0xd4, 0xcd, 0x2c, 0x6e, - 0x70, 0xbc, 0x47, 0xd0, 0x40, 0x4f, 0x4f, 0x82, 0x99, 0x8e, 0xcd, 0xba, 0x2d, 0xb6, 0xff, 0x59, - 0xd0, 0xf9, 0xf5, 0x42, 0x24, 0xd7, 0x5c, 0xfc, 0x7e, 0x21, 0xe6, 0x29, 0x62, 0x4b, 0xb4, 0xaa, - 0x65, 0x22, 0xb0, 0x6a, 0x87, 0xef, 0x82, 0x64, 0x2c, 0x91, 0x72, 0x78, 0x46, 0x61, 0xac, 0x1a, - 0xf3, 0x39, 0xc5, 0xda, 0xe4, 0x26, 0x8b, 0xea, 0x5d, 0x4c, 0xe2, 0x54, 0x05, 0x93, 0x51, 0xac, - 0x0f, 0xf7, 0x8e, 0xae, 0x46, 0xd1, 0x62, 0x2c, 0x78, 0xbc, 0x94, 0xbb, 0xa9, 0x39, 0xf3, 0x32, - 0x9b, 0xfd, 0x00, 0x9b, 0x1b, 0xb1, 0x54, 0x6b, 0x6a, 0x90, 0x62, 0x89, 0xcb, 0xf6, 0xa1, 0x73, - 0x34, 0xb9, 0x10, 0xe3, 0xb1, 0x18, 0x0f, 0x82, 0x34, 0x70, 0x9b, 0x55, 0x03, 0x44, 0x41, 0xc5, - 0xfb, 0x60, 0xc1, 0x46, 0x16, 0xfd, 0x7c, 0x16, 0x4f, 0xe7, 0x02, 0x53, 0x7c, 0x94, 0x24, 0x2a, - 0xc5, 0x47, 0x49, 0xc2, 0x1e, 0x41, 0x83, 0x8b, 0xf9, 0x22, 0x4a, 0x55, 0x95, 0x7c, 0xa9, 0x2d, - 0xaa, 0xbd, 0x8b, 0x28, 0xe5, 0x4a, 0x8b, 0xfd, 0x1c, 0x36, 0x0b, 0x75, 0xa8, 0x9e, 0x85, 0xef, - 0xe8, 0x7d, 0x05, 0x39, 0x2f, 0xa9, 0x7b, 0x7f, 0xa9, 0x43, 0xdb, 0xb0, 0x9c, 0x17, 0x19, 0xe2, - 0xb3, 0x91, 0x15, 0xd9, 0xd7, 0x34, 0xa7, 0xdd, 0x30, 0xf5, 0x60, 0x4f, 0xea, 0x80, 0x75, 0x9a, - 0x95, 0xa5, 0x75, 0xaa, 0x1b, 0xa1, 0x7d, 0x5b, 0x23, 0xc4, 0xa9, 0xef, 0x5d, 0x30, 0xbd, 0x14, - 0x63, 0x2a, 0xcb, 0x26, 0x57, 0x24, 0xdb, 0xd3, 0x5d, 0x81, 0xf2, 0x58, 0xe8, 0x35, 0x4a, 0xc2, - 0x75, 0xe7, 0x90, 0x5d, 0x0e, 0x27, 0x83, 0x86, 0xac, 0x17, 0x49, 0xb1, 0x67, 0xd0, 0xd6, 0xed, - 0x6b, 0x9e, 0xa5, 0xa8, 0xab, 0x4d, 0x69, 0x21, 0x37, 0x15, 0xd9, 0x8b, 0xf2, 0x88, 0xe6, 0xb6, - 0xc8, 0x0b, 0xb7, 0x10, 0xb9, 0x21, 0xe7, 0xe5, 0x91, 0x6e, 0xdf, 0x98, 0x19, 0x5d, 0xa0, 0xcd, - 0xf7, 0xf5, 0xe6, 0x5c, 0xc4, 0x8d, 0xc9, 0xf2, 0x89, 0xf9, 0x96, 0xb8, 0x6d, 0xda, 0xd3, 0x2d, - 0x22, 0x27, 0x65, 0xdc, 0x7c, 0x73, 0xf6, 0x8d, 0x87, 0xcc, 0xed, 0x94, 0x0f, 0xca, 0x45, 0xdc, - 0x78, 0xee, 0xfc, 0x8a, 0xf9, 0xce, 0xdd, 0xa0, 0xad, 0xd5, 0xc3, 0x9b, 0x54, 0xe1, 0x15, 0x53, - 0xe1, 0x8b, 0xf2, 0x24, 0xe0, 0x6e, 0x96, 0x81, 0x2a, 0xca, 0x79, 0x79, 0x72, 0xd8, 0x37, 0x86, - 0x71, 0xf7, 0x5e, 0xd9, 0xff, 0x5c, 0xc4, 0xb5, 0x96, 0xf7, 0xb7, 0x1a, 0x6c, 0xf8, 0x93, 0x59, - 0x9c, 0xa4, 0x46, 0x17, 0xf1, 0xa7, 0x63, 0x71, 0xa5, 0xba, 0x08, 0x11, 0xd5, 0x0f, 0x2d, 0x75, - 0x73, 0xec, 0x26, 0xd4, 0x3d, 0x1c, 0x2e, 0x09, 0xa3, 0x82, 0x9c, 0x42, 0x05, 0x6d, 0x43, 0x4b, - 0x5e, 0x17, 0x14, 0xd5, 0x49, 0xa4, 0x19, 0xf2, 0x1b, 0x63, 0x49, 0xb3, 0x66, 0x83, 0xa6, 0x57, - 0x45, 0x62, 0xe7, 0x94, 0x6a, 0x24, 0x6c, 0x92, 0xd0, 0xe0, 0xa0, 0xfc, 0x2c, 0x9c, 0x88, 0x79, - 0x1a, 0x4c, 0x66, 0xd8, 0x8a, 0xec, 0xbe, 0xcd, 0x0d, 0x0e, 0x76, 0x21, 0x0a, 0xe2, 0x65, 0x22, - 0x82, 0x54, 0x8c, 0x0f, 0x52, 0xaa, 0x40, 0x9b, 0x97, 0xb8, 0xa8, 0x47, 0x61, 0x69, 0x3d, 0x90, - 0x7a, 0x45, 0x2e, 0xbd, 0xa4, 0x91, 0x08, 0x12, 0xaa, 0xab, 0x26, 0x97, 0x84, 0xf7, 0xcf, 0x1a, - 0x30, 0x89, 0xa4, 0x9c, 0x15, 0xbf, 0x35, 0x38, 0x6f, 0x87, 0xad, 0x08, 0x4e, 0x63, 0x05, 0x9c, - 0xaf, 0xf2, 0x09, 0x57, 0x02, 0x93, 0x51, 0xd8, 0xfe, 0xf5, 0xe3, 0x23, 0x51, 0xb5, 0xb8, 0xc9, - 0x62, 0x1e, 0x74, 0x8c, 0x97, 0x0f, 0xaf, 0x2d, 0xda, 0x2e, 0xf0, 0x2a, 0xa0, 0x85, 0x3b, 0x42, - 0xdb, 0xbe, 0x1d, 0xda, 0x8e, 0x09, 0xed, 0x07, 0x0b, 0x3a, 0x07, 0x69, 0x3c, 0x09, 0x47, 0x5c, - 0x8c, 0xe2, 0x64, 0x7c, 0x33, 0xa8, 0x12, 0xbe, 0x9a, 0x09, 0xdf, 0x1e, 0xd8, 0xfe, 0xfb, 0x24, - 0xeb, 0x9e, 0xdb, 0xc6, 0x6c, 0xb6, 0x92, 0x2b, 0x8e, 0x8a, 0xec, 0x1b, 0xa8, 0xf9, 0x09, 0x55, - 0x6e, 0xa1, 0xef, 0x17, 0x2e, 0x09, 0xaf, 0xf9, 0x89, 0xf7, 0x63, 0xe8, 0x4a, 0xa7, 0x94, 0x28, - 0x7b, 0x87, 0xba, 0x50, 0x3f, 0x4a, 0x92, 0x58, 0xbd, 0x44, 0x92, 0xf0, 0xae, 0xa0, 0x7b, 0x96, - 0x04, 0xd3, 0x79, 0x14, 0xa4, 0x02, 0x13, 0xf3, 0x39, 0xf5, 0x51, 0xf5, 0x01, 0xdf, 0x83, 0xf6, - 0x69, 0x9c, 0xbe, 0x4d, 0xc2, 0x94, 0x5a, 0x86, 0x6c, 0xfe, 0x26, 0xcb, 0xfb, 0x21, 0x7c, 0x59, - 0x3a, 0x59, 0x3f, 0x98, 0x58, 0x52, 0xb6, 0xfe, 0xf0, 0x1d, 0xc2, 0xfd, 0x5c, 0xd5, 0x1f, 0x7c, - 0x96, 0x8f, 0xab, 0x46, 0x7f, 0x64, 0x44, 0x4e, 0x46, 0xb3, 0xe3, 0x2b, 0xa2, 0xf1, 0x0e, 0xc1, - 0xcd, 0xd0, 0x94, 0xff, 0x17, 0x32, 0x0f, 0xce, 0x43, 0xb1, 0xbc, 0xe9, 0x93, 0x8a, 0x06, 0x86, - 0x1a, 0xfd, 0x95, 0xa0, 0xb5, 0xf7, 0x5f, 0x0b, 0xba, 0x55, 0x46, 0x74, 0x71, 0x59, 0x46, 0x71, - 0xb1, 0xe7, 0x50, 0x7f, 0x1f, 0x8a, 0xa5, 0x1a, 0x11, 0xbc, 0x95, 0x94, 0xaf, 0x78, 0xc2, 0xe5, - 0x06, 0xbc, 0x5a, 0x07, 0xa3, 0x34, 0x8c, 0xa7, 0xea, 0x7b, 0x40, 0x52, 0x78, 0xce, 0x61, 0x14, - 0x8f, 0x7e, 0x27, 0xbf, 0x74, 0xb9, 0x24, 0x2a, 0xae, 0x4a, 0xfd, 0x8e, 0x57, 0x65, 0xbd, 0xea, - 0xaa, 0x78, 0x7f, 0xb5, 0x14, 0x56, 0xc6, 0xcc, 0xf6, 0xc9, 0x8c, 0xe9, 0x0b, 0x62, 0xab, 0x0b, - 0xe2, 0xca, 0xc1, 0x53, 0xcf, 0xd7, 0x8a, 0xc4, 0x61, 0x17, 0x97, 0xf4, 0x9b, 0xc3, 0xa1, 0x2c, - 0xe5, 0xf4, 0x27, 0xba, 0xd2, 0x6a, 0xb0, 0xeb, 0x55, 0xc1, 0x1e, 0x6e, 0xfd, 0xfd, 0xe3, 0x8e, - 0xf5, 0x8f, 0x8f, 0x3b, 0xd6, 0xbf, 0x3f, 0xee, 0x58, 0x7f, 0xfa, 0xcf, 0xce, 0xda, 0xc5, 0x3a, - 0xfd, 0xd6, 0xfa, 0xc9, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0xf4, 0x59, 0xf3, 0x5b, 0xe6, 0x12, - 0x00, 0x00, + 0xe9, 0x21, 0x36, 0xb0, 0xf4, 0xd9, 0x13, 0x14, 0xcb, 0x82, 0x44, 0x6f, 0x6c, 0x9e, 0x95, 0xcc, + 0x02, 0x9a, 0x12, 0xba, 0x78, 0xa9, 0x0d, 0x58, 0x86, 0x01, 0xe4, 0x62, 0x5b, 0x19, 0xa8, 0x38, + 0x89, 0xc0, 0x6b, 0xcb, 0xe3, 0xa5, 0x86, 0x24, 0xa3, 0xd8, 0x77, 0xd5, 0x29, 0x0e, 0xc5, 0x7c, + 0xcf, 0xb8, 0x4a, 0xe8, 0x85, 0x3a, 0xf6, 0x57, 0x00, 0x3f, 0x4b, 0xe2, 0xc5, 0x8c, 0x40, 0x63, + 0x7d, 0xa8, 0x13, 0x95, 0xc5, 0xc7, 0xf4, 0x26, 0xe5, 0x1b, 0x97, 0x0a, 0xd5, 0xa0, 0x63, 0x72, + 0x86, 0x8b, 0x89, 0xbc, 0x69, 0x1c, 0x97, 0x58, 0x4a, 0xcd, 0xf3, 0x20, 0xca, 0xc5, 0xe7, 0x41, + 0x94, 0xc5, 0x8d, 0xcb, 0xa2, 0x19, 0x5b, 0x99, 0x79, 0x00, 0xcd, 0x57, 0x51, 0x1c, 0xa4, 0xa8, + 0x8c, 0xb6, 0x2c, 0x9e, 0xd3, 0x6c, 0x1f, 0x60, 0x20, 0x46, 0xe1, 0x24, 0x88, 0x50, 0xea, 0x94, + 0x1b, 0x40, 0x26, 0xe3, 0x86, 0x92, 0xf7, 0x14, 0x1a, 0x19, 0x55, 0x8d, 0x3d, 0x72, 0x87, 0xa3, + 0x20, 0x12, 0xca, 0x0b, 0x22, 0xbc, 0xb7, 0xb0, 0x21, 0x8b, 0x11, 0x9f, 0xa6, 0xa1, 0x48, 0xef, + 0x50, 0x8a, 0x77, 0x7a, 0xe4, 0xbc, 0x3f, 0x59, 0xe0, 0xe0, 0x4a, 0x19, 0xb0, 0xb4, 0x01, 0xf3, + 0x36, 0x3a, 0xf2, 0x36, 0xb2, 0x1e, 0xb4, 0x87, 0x29, 0xbe, 0x81, 0xba, 0x8d, 0xb5, 0xb8, 0xc9, + 0x42, 0xbc, 0xfc, 0x69, 0xaa, 0xd3, 0x6d, 0xf3, 0x9c, 0x66, 0xdb, 0xd0, 0xc2, 0xde, 0x24, 0x85, + 0xd8, 0xc8, 0x9a, 0x5c, 0x33, 0xd8, 0x0e, 0x80, 0x42, 0x76, 0x21, 0xa8, 0x9b, 0x59, 0xdc, 0xe0, + 0x78, 0x8f, 0xa0, 0x81, 0x9e, 0x9e, 0x04, 0x33, 0x1d, 0x9b, 0x75, 0x5b, 0x6c, 0xff, 0xb5, 0xa0, + 0xf3, 0xcb, 0x85, 0x48, 0xae, 0xb9, 0xf8, 0xed, 0x42, 0xcc, 0x53, 0xc4, 0x96, 0x68, 0x55, 0xcb, + 0x44, 0x60, 0xd5, 0x0e, 0xdf, 0x05, 0xc9, 0x58, 0x22, 0xe5, 0xf0, 0x8c, 0xc2, 0x58, 0x35, 0xe6, + 0x73, 0x8a, 0xb5, 0xc9, 0x4d, 0x16, 0xd5, 0xbb, 0x98, 0xc4, 0xa9, 0x0a, 0x26, 0xa3, 0x58, 0x1f, + 0xee, 0x1d, 0x5d, 0x8d, 0xa2, 0xc5, 0x58, 0xf0, 0x78, 0x29, 0x77, 0x53, 0x73, 0xe6, 0x65, 0x36, + 0xfb, 0x1e, 0x36, 0x37, 0x62, 0xa9, 0xd6, 0xd4, 0x20, 0xc5, 0x12, 0x97, 0xed, 0x43, 0xe7, 0x68, + 0x72, 0x21, 0xc6, 0x63, 0x31, 0x1e, 0x04, 0x69, 0xe0, 0x36, 0xab, 0x06, 0x88, 0x82, 0x8a, 0xf7, + 0xc1, 0x82, 0x8d, 0x2c, 0xfa, 0xf9, 0x2c, 0x9e, 0xce, 0x05, 0xa6, 0xf8, 0x28, 0x49, 0x54, 0x8a, + 0x8f, 0x92, 0x84, 0x3d, 0x82, 0x06, 0x17, 0xf3, 0x45, 0x94, 0xaa, 0x2a, 0xf9, 0x52, 0x5b, 0x54, + 0x7b, 0x17, 0x51, 0xca, 0x95, 0x16, 0xfb, 0x29, 0x6c, 0x16, 0xea, 0x50, 0x3d, 0x0b, 0xdf, 0xd2, + 0xfb, 0x0a, 0x72, 0x5e, 0x52, 0xf7, 0xfe, 0x5c, 0x87, 0xb6, 0x61, 0x39, 0x2f, 0x32, 0xc4, 0x67, + 0x23, 0x2b, 0xb2, 0xaf, 0x69, 0xa6, 0xbb, 0x61, 0xea, 0xc1, 0x9e, 0xd4, 0x01, 0xeb, 0x34, 0x2b, + 0x4b, 0xeb, 0x54, 0x37, 0x42, 0xfb, 0xb6, 0x46, 0x88, 0x13, 0xe2, 0xbb, 0x60, 0x7a, 0x29, 0xc6, + 0x54, 0x96, 0x4d, 0xae, 0x48, 0xb6, 0xa7, 0xbb, 0x02, 0xe5, 0xb1, 0xd0, 0x6b, 0x94, 0x84, 0xeb, + 0xce, 0x21, 0xbb, 0x1c, 0x4e, 0x06, 0x0d, 0x59, 0x2f, 0x92, 0x62, 0xcf, 0xa0, 0xad, 0xdb, 0xd7, + 0x3c, 0x4b, 0x51, 0x57, 0x9b, 0xd2, 0x42, 0x6e, 0x2a, 0xb2, 0x17, 0xe5, 0x11, 0xcd, 0x6d, 0x91, + 0x17, 0x6e, 0x21, 0x72, 0x43, 0xce, 0xcb, 0x23, 0xdd, 0xbe, 0x31, 0x33, 0xba, 0x40, 0x9b, 0xef, + 0xeb, 0xcd, 0xb9, 0x88, 0x1b, 0x93, 0xe5, 0x13, 0xf3, 0x2d, 0x71, 0xdb, 0xb4, 0xa7, 0x5b, 0x44, + 0x4e, 0xca, 0xb8, 0xf9, 0xe6, 0xec, 0x1b, 0x0f, 0x99, 0xdb, 0x29, 0x1f, 0x94, 0x8b, 0xb8, 0xf1, + 0xdc, 0xf9, 0x15, 0xf3, 0x9d, 0xbb, 0x41, 0x5b, 0xab, 0x87, 0x37, 0xa9, 0xc2, 0x2b, 0xa6, 0xc2, + 0x17, 0xe5, 0x49, 0xc0, 0xdd, 0x2c, 0x03, 0x55, 0x94, 0xf3, 0xf2, 0xe4, 0xb0, 0x6f, 0x0c, 0xe3, + 0xee, 0xbd, 0xb2, 0xff, 0xb9, 0x88, 0x6b, 0x2d, 0xef, 0xaf, 0x35, 0xd8, 0xf0, 0x27, 0xb3, 0x38, + 0x49, 0x8d, 0x2e, 0x22, 0xa7, 0x7f, 0xab, 0x72, 0xfa, 0xaf, 0x95, 0xde, 0x49, 0xea, 0x26, 0xd4, + 0x3d, 0x1c, 0x2e, 0x09, 0xa3, 0x82, 0x9c, 0x42, 0x05, 0x6d, 0x43, 0x4b, 0x5e, 0x17, 0x14, 0xd5, + 0x49, 0xa4, 0x19, 0xf2, 0x7b, 0x64, 0x49, 0xb3, 0x66, 0x83, 0xa6, 0x57, 0x45, 0x62, 0xe7, 0x94, + 0x6a, 0x24, 0x6c, 0x92, 0xd0, 0xe0, 0xa0, 0xfc, 0x2c, 0x9c, 0x88, 0x79, 0x1a, 0x4c, 0x66, 0xd8, + 0x8a, 0xec, 0xbe, 0xcd, 0x0d, 0x0e, 0x76, 0x21, 0x0a, 0xe2, 0x65, 0x22, 0x82, 0x54, 0x8c, 0x0f, + 0x52, 0xaa, 0x40, 0x9b, 0x97, 0xb8, 0xa8, 0x47, 0x61, 0x69, 0x3d, 0x90, 0x7a, 0x45, 0x2e, 0xbd, + 0xa4, 0x91, 0x08, 0x12, 0xaa, 0xab, 0x26, 0x97, 0x84, 0xf7, 0x8f, 0x1a, 0x30, 0x89, 0xa4, 0x9c, + 0x15, 0xff, 0x6f, 0x70, 0xde, 0x0e, 0x5b, 0x11, 0x9c, 0xc6, 0x0a, 0x38, 0x5f, 0xe5, 0x13, 0xae, + 0x04, 0x26, 0xa3, 0xb0, 0xfd, 0xeb, 0xc7, 0x47, 0xa2, 0x6a, 0x71, 0x93, 0xc5, 0x3c, 0xe8, 0x18, + 0x2f, 0x1f, 0x5e, 0x5b, 0xb4, 0x5d, 0xe0, 0x55, 0x40, 0x0b, 0x77, 0x84, 0xb6, 0x7d, 0x3b, 0xb4, + 0x1d, 0x13, 0xda, 0x0f, 0x16, 0x74, 0x0e, 0xd2, 0x78, 0x12, 0x8e, 0xb8, 0x18, 0xc5, 0xc9, 0xf8, + 0x66, 0x50, 0x25, 0x7c, 0x35, 0x13, 0xbe, 0x3d, 0xb0, 0xfd, 0xf7, 0x49, 0xd6, 0x3d, 0xb7, 0x8d, + 0xd9, 0x6c, 0x25, 0x57, 0x1c, 0x15, 0xd9, 0x37, 0x50, 0xf3, 0x13, 0xaa, 0xdc, 0x42, 0xdf, 0x2f, + 0x5c, 0x12, 0x5e, 0xf3, 0x13, 0xef, 0x87, 0xd0, 0x95, 0x4e, 0x29, 0x51, 0xf6, 0x0e, 0x75, 0xa1, + 0x7e, 0x94, 0x24, 0xb1, 0x7a, 0x89, 0x24, 0xe1, 0x5d, 0x41, 0xf7, 0x2c, 0x09, 0xa6, 0xf3, 0x28, + 0x48, 0x05, 0x26, 0xe6, 0x73, 0xea, 0xa3, 0xea, 0x63, 0xbf, 0x07, 0xed, 0xd3, 0x38, 0x7d, 0x9b, + 0x84, 0x29, 0xb5, 0x0c, 0xd9, 0xfc, 0x4d, 0x96, 0xf7, 0x7d, 0xf8, 0xb2, 0x74, 0xb2, 0x7e, 0x30, + 0xb1, 0xa4, 0x6c, 0xfd, 0xe1, 0x3b, 0x84, 0xfb, 0xb9, 0xaa, 0x3f, 0xf8, 0x2c, 0x1f, 0x57, 0x8d, + 0xfe, 0xc0, 0x88, 0x9c, 0x8c, 0x66, 0xc7, 0x57, 0x44, 0xe3, 0x1d, 0x82, 0x9b, 0xa1, 0x29, 0xff, + 0x45, 0x64, 0x1e, 0x9c, 0x87, 0x62, 0x79, 0xd3, 0x27, 0x15, 0x0d, 0x0c, 0x35, 0xfa, 0x83, 0x41, + 0x6b, 0xef, 0x3f, 0x16, 0x74, 0xab, 0x8c, 0xe8, 0xe2, 0xb2, 0x8c, 0xe2, 0x62, 0xcf, 0xa1, 0xfe, + 0x3e, 0x14, 0x4b, 0x35, 0x22, 0x78, 0x2b, 0x29, 0x5f, 0xf1, 0x84, 0xcb, 0x0d, 0x78, 0xb5, 0x0e, + 0x46, 0x69, 0x18, 0x4f, 0xd5, 0xf7, 0x80, 0xa4, 0xf0, 0x9c, 0xc3, 0x28, 0x1e, 0xfd, 0x46, 0x7e, + 0xe9, 0x72, 0x49, 0x54, 0x5c, 0x95, 0xfa, 0x1d, 0xaf, 0xca, 0x7a, 0xd5, 0x55, 0xf1, 0xfe, 0x62, + 0x29, 0xac, 0x8c, 0x99, 0xed, 0x93, 0x19, 0xd3, 0x17, 0xc4, 0x56, 0x17, 0xc4, 0x95, 0x83, 0xa7, + 0x9e, 0xaf, 0x15, 0x89, 0xc3, 0x2e, 0x2e, 0xe9, 0x37, 0x87, 0x43, 0x59, 0xca, 0xe9, 0x4f, 0x74, + 0xa5, 0xd5, 0x60, 0xd7, 0xab, 0x82, 0x3d, 0xdc, 0xfa, 0xdb, 0xc7, 0x1d, 0xeb, 0xef, 0x1f, 0x77, + 0xac, 0x7f, 0x7d, 0xdc, 0xb1, 0xfe, 0xf0, 0xef, 0x9d, 0xb5, 0x8b, 0x75, 0xfa, 0x07, 0xf6, 0xa3, + 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x2d, 0xb9, 0x97, 0xfb, 0x13, 0x13, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -2800,6 +2816,20 @@ func (m *Row) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0x32 + } + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0x2a + } if len(m.Roaring) > 0 { i -= len(m.Roaring) copy(dAtA[i:], m.Roaring) @@ -5183,6 +5213,14 @@ func (m *Row) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -6457,6 +6495,70 @@ func (m *Row) Unmarshal(dAtA []byte) error { m.Roaring = []byte{} } iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index 6d6a56d70..9e3b4241e 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -7,6 +7,8 @@ message Row { repeated string Keys = 3; repeated Attr Attrs = 2; bytes Roaring = 4; + string Index = 5; + string Field = 6; } message RowMatrix { diff --git a/row.go b/row.go index 9170926a7..5e2256841 100644 --- a/row.go +++ b/row.go @@ -33,6 +33,15 @@ type Row struct { // Attributes associated with the row. Attrs map[string]interface{} + + // Index tells what index this row is from - needed for key translation. + Index string + + // Field tells what field this row is from if it's a "vertical" + // row. It may be the result of a Distinct query or Rows + // query. Knowing the index and field, we can figure out how to + // interpret the row data. + Field string } // NewRow returns a new instance of Row. @@ -322,7 +331,7 @@ func (r *Row) Union(others ...*Row) *Row { output = append(output, *toProcess[0].Union(toProcess[1:]...)) } } - return &Row{segments: output} + return &Row{Index: r.Index, Field: r.Field, segments: output} } // Difference returns the diff of r and other. From 6ff6fa7bb8cdf7eae30bdc207b0c5d0ac79c9866 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 23 Dec 2020 14:35:57 -0600 Subject: [PATCH 04/13] fix comments/capitalization --- executor.go | 11 ++++++----- executor_test.go | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/executor.go b/executor.go index 4a0fa5d27..39bfa30f8 100644 --- a/executor.go +++ b/executor.go @@ -361,7 +361,7 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr } // handlePreCalls traverses the call tree looking for calls that need -// precomputed values (e.g. Distinct, UnionRows, ConstRow...) +// precomputed values (e.g. Distinct, UnionRows, ConstRow...). func (e *executor) handlePreCalls(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) error { if c.Name == "Precomputed" { idx := c.Args["valueidx"].(int64) @@ -4264,8 +4264,8 @@ func (e *executor) executeCount(ctx context.Context, qcx *Qcx, index string, c * child := c.Children[0] - // if the child is precomputed, we'll bypass mapreduce, ignore - // shards, and just count the number of bits + // If the child is precomputed, we'll bypass mapreduce, ignore + // shards, and just count the number of bits. if child.Name == "Precomputed" { count := uint64(0) for _, irow := range child.Precomputed { @@ -6056,7 +6056,7 @@ func (e *executor) howToTranslate(idx *Index, row *Row) (rowIdx *Index, rowField } } - // Handle the case where the Row has specified a field + // Handle the case where the Row has specified a field. if rowField != nil { // Handle case where field has a foreign index. if rowField.ForeignIndex() != "" { @@ -6073,7 +6073,8 @@ func (e *executor) howToTranslate(idx *Index, row *Row) (rowIdx *Index, rowField return rowIdx, rowField, noTranslation, nil } - // In this case, the row has specified an index, but not a field, so we translate according to that index. + // In this case, the row has specified an index, but not a field, + // so we translate according to that index. if rowIdx != idx && rowIdx.Keys() { return rowIdx, rowField, byRowIndex, nil } diff --git a/executor_test.go b/executor_test.go index 5d79298ab..89b25ea40 100644 --- a/executor_test.go +++ b/executor_test.go @@ -6743,7 +6743,17 @@ func TestMissingKeyRegression(t *testing.T) { } } -func TestDistinctOnSetsKeyedIndex(t *testing.T) { +// TestVariousQueries has originally been written to test out a +// variety of scenarios with Distinct, but it's structure is more +// general purpose. My vision is to eventually have any test which +// needs to test a single query be in here, and have a robust enough +// test data set loaded at the start which covers what we want to +// test. +// +// I'd also like to have it automatically run a matrix of scenarios +// (single and multi-node clusters, different endpoints for the +// queries (HTTP, GRPC, Postgres), etc.). +func TestVariousQueries(t *testing.T) { c := test.MustRunCluster(t, 3) defer c.Close() @@ -6858,11 +6868,12 @@ func TestDistinctOnSetsKeyedIndex(t *testing.T) { }, }, - // handling this case properly will require changing the way + // Handling this case properly will require changing the way // that precomputed data is stored on Call objects. Currently // if a Distinct is at all nested (e.g. within a Count) it // gets handled by executor.handlePreCalls which assumes that // only the positive values are worthwhile. + // // { // query: "Count(Distinct(field=affinity))", // verifier: func(t *testing.T, resp pilosa.QueryResponse) { From 427e9cb5387411778df0a7da3730e562bdba4bab Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 23 Dec 2020 14:52:14 -0600 Subject: [PATCH 05/13] fix executor tests which were expecting a signedrow from Distinct --- executor_test.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/executor_test.go b/executor_test.go index 89b25ea40..76174d4b6 100644 --- a/executor_test.go +++ b/executor_test.go @@ -5434,9 +5434,9 @@ func TestExecutor_ForeignIndex(t *testing.T) { if !sameStringSlice(distinct.Pos.Keys, []string{"one", "two", "twenty-one"}) { t.Fatalf("unexpected keys: %v", distinct.Pos.Keys) } - distinct = c.Query(t, "child", `Distinct(index="child", field="parent_set_id")`).Results[0].(pilosa.SignedRow) - if !sameStringSlice(distinct.Pos.Keys, []string{"one", "two", "twenty-one"}) { - t.Fatalf("unexpected keys: %v", distinct.Pos.Keys) + row := c.Query(t, "child", `Distinct(index="child", field="parent_set_id")`).Results[0].(*pilosa.Row) + if !sameStringSlice(row.Keys, []string{"one", "two", "twenty-one"}) { + t.Fatalf("unexpected keys: %v", row.Keys) } eq := c.Query(t, "child", `Row(parent_id=="one")`).Results[0].(*pilosa.Row) @@ -6512,7 +6512,6 @@ func TestExecutor_BareDistinct(t *testing.T) { c.CreateField(t, "i", pilosa.IndexOptions{}, "ints", pilosa.OptFieldTypeInt(0, math.MaxInt64), ) - c.CreateField(t, "i", pilosa.IndexOptions{}, "set") c.CreateField(t, "i", pilosa.IndexOptions{}, "filter") // Populate integer data. @@ -6521,17 +6520,13 @@ func TestExecutor_BareDistinct(t *testing.T) { Set(%d, ints=2) `, ShardWidth)) c.Query(t, "i", fmt.Sprintf(` - Set(0, set=1) - Set(1, set=2) - Set(%d, set=2) Set(0, filter=1) Set(%d, filter=1) - `, 65537, 65537)) + `, 65537)) for _, pql := range []string{ `Distinct(field="ints")`, `Distinct(index="i", field="ints")`, - `Distinct(Row(filter=1), field="set")`, } { exp := []uint64{1, 2} res := c.Query(t, "i", pql).Results[0].(pilosa.SignedRow) @@ -6890,6 +6885,14 @@ func TestVariousQueries(t *testing.T) { } }, }, + { + query: "Distinct(Row(affinity<0), field=likes)", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"pilosa", "zebra", "icecream"}) { + t.Errorf("wrong values: %+v", resp.Results[0]) + } + }, + }, } for i, tst := range tests { From 1372bafe026d64ad56c446e65e0ffb95e46af227 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 23 Dec 2020 18:53:17 -0600 Subject: [PATCH 06/13] fix bugs where row index and field weren't always being propagated I used a "paranoia" check to find these, but then realized the check had a ton of false positives and doing it properly wasn't going to be straightforward. I'm leaving the paranoia stuff in unless there are objections, because I've wanted it before and not had it. I also removed a log line that is very verbose and I don't think helps anyone. --- Makefile | 2 +- executor.go | 10 ++++++++-- executor_test.go | 40 ++++++++++++++++++++++++++++++++++++---- nop_paranoia.go | 19 +++++++++++++++++++ paranoia.go | 19 +++++++++++++++++++ row.go | 7 +++++++ server.go | 1 - 7 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 nop_paranoia.go create mode 100644 paranoia.go diff --git a/Makefile b/Makefile index abd07304d..d622a235c 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) BUILD_TAGS += $(if $(RELEASE_ENABLED),release) BUILD_TAGS += shardwidth$(SHARD_WIDTH) -TEST_TAGS = roaringparanoia +TEST_TAGS = roaringparanoia paranoia define LICENSE_HASH_CODE head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " " endef diff --git a/executor.go b/executor.go index 39bfa30f8..6694deeb1 100644 --- a/executor.go +++ b/executor.go @@ -1421,7 +1421,10 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str } bsig := field.bsiGroup(fieldName) if bsig == nil { - result = new(Row) + result = &Row{ + Index: index, + Field: fieldName, + } } else { result = SignedRow{} } @@ -5188,7 +5191,10 @@ func makeEmbeddedDataForShards(allRows []*Row, shards []uint64) []*Row { } segments := row.segments segmentIndex := 0 - newRows[i] = &Row{} + newRows[i] = &Row{ + Index: row.Index, + Field: row.Field, + } for _, shard := range shards { for segmentIndex < len(segments) && segments[segmentIndex].shard < shard { segmentIndex++ diff --git a/executor_test.go b/executor_test.go index 76174d4b6..e065004ef 100644 --- a/executor_test.go +++ b/executor_test.go @@ -6844,7 +6844,7 @@ func TestVariousQueries(t *testing.T) { }, }, { - query: "Distinct(Row(affinity>=0), field=affinity)", + query: "Distinct(Row(affinity>=0),field=affinity)", verifier: func(t *testing.T, resp pilosa.QueryResponse) { if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Pos.Columns(), []uint64{0, 5, 10}) { t.Errorf("wrong positive records: %+v", resp.Results[0].(pilosa.SignedRow).Pos.Columns()) @@ -6855,7 +6855,7 @@ func TestVariousQueries(t *testing.T) { }, }, { - query: "Count(Distinct(Row(affinity>=0), field=affinity))", + query: "Count(Distinct(Row(affinity>=0),field=affinity))", verifier: func(t *testing.T, resp pilosa.QueryResponse) { if resp.Results[0].(uint64) != 3 { t.Errorf("wrong number of values: %+v", resp.Results[0]) @@ -6877,6 +6877,30 @@ func TestVariousQueries(t *testing.T) { // } // }, // }, + { + query: "Distinct(Row(affinity<0),field=likes)", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"pilosa", "zebra", "icecream"}) { + t.Errorf("wrong values: %+v", resp.Results[0]) + } + }, + }, + { + query: "Distinct(Row(affinity>0),field=likes)", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pangolin", "icecream"}) { + t.Errorf("wrong values: %+v", resp.Results[0]) + } + }, + }, + { + query: "Distinct(Row(likenums=1),field=likes)", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "icecream"}) { + t.Errorf("wrong values: %+v", resp.Results[0]) + } + }, + }, { query: "Distinct(field=likes)", verifier: func(t *testing.T, resp pilosa.QueryResponse) { @@ -6886,9 +6910,17 @@ func TestVariousQueries(t *testing.T) { }, }, { - query: "Distinct(Row(affinity<0), field=likes)", + query: "Distinct(All(),field=likes)", verifier: func(t *testing.T, resp pilosa.QueryResponse) { - if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"pilosa", "zebra", "icecream"}) { + if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pilosa", "pangolin", "zebra", "toucan", "dog", "icecream"}) { + t.Errorf("wrong values: %+v", resp.Results[0]) + } + }, + }, + { + query: "Distinct(field=likes )", + verifier: func(t *testing.T, resp pilosa.QueryResponse) { + if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pilosa", "pangolin", "zebra", "toucan", "dog", "icecream"}) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, diff --git a/nop_paranoia.go b/nop_paranoia.go new file mode 100644 index 000000000..77cdae411 --- /dev/null +++ b/nop_paranoia.go @@ -0,0 +1,19 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !paranoia + +package pilosa + +const paranoia = false diff --git a/paranoia.go b/paranoia.go new file mode 100644 index 000000000..fb717db91 --- /dev/null +++ b/paranoia.go @@ -0,0 +1,19 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build paranoia + +package pilosa + +const paranoia = true diff --git a/row.go b/row.go index 5e2256841..98f5c9275 100644 --- a/row.go +++ b/row.go @@ -70,6 +70,8 @@ func (r *Row) Clone() (clone *Row) { clone = &Row{ Keys: keyClone, Attrs: attrClone, + Index: r.Index, + Field: r.Field, } for _, seg := range r.segments { @@ -299,6 +301,11 @@ func (r *Row) Union(others ...*Row) *Row { toProcess := make([]*rowSegment, 0, len(others)+1) var output []rowSegment for _, other := range others { + if paranoia { // nolint:staticcheck + // TODO I think there is a good check we can do here, but + // it's nontrivial to check whether two rows are + // compatible because of foreign indexes and such. + } if len(other.segments) > 0 { segments = append(segments, other.segments) } diff --git a/server.go b/server.go index 89d197fef..b664e4c87 100644 --- a/server.go +++ b/server.go @@ -657,7 +657,6 @@ func (s *Server) monitorResetTranslationSync() { case <-s.closing: return case <-s.resetTranslationSyncCh: - s.logger.Printf("holder translation sync beginning") s.wg.Add(1) go func() { // Obtaining this lock ensures that there is only From 446950979a8f1de2d9ff8cb09e3cf728361f5423 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 28 Dec 2020 11:00:46 -0600 Subject: [PATCH 07/13] fix potential nil dereference in SignedRow.ToRows This used to be possible to hit, but I think now that Distinct on a set field returns a *Row rather than a SignedRow it isn't an issue. (I wasn't able to trigger it in the tests). Adding the fix anyway as it seems safer than not. The rest of the changes are test infrastructure to make it easy to call GRPC queries and verify the results as CSV. --- executor.go | 58 ++++++++++++---------- executor_test.go | 120 +++++++++++++++++++++++++++++++++++++++------ proto/interface.go | 50 +++++++++++++++++++ server.go | 4 ++ test/cluster.go | 29 +++++++++++ test/pilosa.go | 4 +- 6 files changed, 220 insertions(+), 45 deletions(-) diff --git a/executor.go b/executor.go index 6694deeb1..190ce07f3 100644 --- a/executor.go +++ b/executor.go @@ -6690,38 +6690,42 @@ func (s SignedRow) ToTable() (*pb.TableResponse, error) { func (s SignedRow) ToRows(callback func(*pb.RowResponse) error) error { ci := []*pb.ColumnInfo{{Name: s.Field(), Datatype: "int64"}} - negs := s.Neg.Columns() - for i := len(negs) - 1; i >= 0; i-- { - val, err := toNegInt64(negs[i]) - if err != nil { - return errors.Wrap(err, "converting uint64 to int64 (negative)") - } + if s.Neg != nil { + negs := s.Neg.Columns() + for i := len(negs) - 1; i >= 0; i-- { + val, err := toNegInt64(negs[i]) + if err != nil { + return errors.Wrap(err, "converting uint64 to int64 (negative)") + } - if err := callback(&pb.RowResponse{ - Headers: ci, - Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: val}}, - }, - }); err != nil { - return errors.Wrap(err, "calling callback") + if err := callback(&pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: val}}, + }, + }); err != nil { + return errors.Wrap(err, "calling callback") + } + ci = nil } - ci = nil } - for _, id := range s.Pos.Columns() { - val, err := toInt64(id) - if err != nil { - return errors.Wrap(err, "converting uint64 to int64 (positive)") - } + if s.Pos != nil { + for _, id := range s.Pos.Columns() { + val, err := toInt64(id) + if err != nil { + return errors.Wrap(err, "converting uint64 to int64 (positive)") + } - if err := callback(&pb.RowResponse{ - Headers: ci, - Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: val}}, - }, - }); err != nil { - return errors.Wrap(err, "calling callback") + if err := callback(&pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: val}}, + }, + }); err != nil { + return errors.Wrap(err, "calling callback") + } + ci = nil } - ci = nil } return nil } diff --git a/executor_test.go b/executor_test.go index e065004ef..ce51d514e 100644 --- a/executor_test.go +++ b/executor_test.go @@ -6797,44 +6797,69 @@ func TestVariousQueries(t *testing.T) { }) tests := []struct { - query string - verifier func(t *testing.T, resp pilosa.QueryResponse) + query string + qrVerifier func(t *testing.T, resp pilosa.QueryResponse) + csvVerifier func(t *testing.T, resp string) }{ { query: "Count(All())", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if resp.Results[0].(uint64) != 6 { t.Errorf("expected 6, got %+v", resp.Results[0]) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "6\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, { query: "Count(Distinct(field=likenums))", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if resp.Results[0].(uint64) != 7 { t.Errorf("wrong count: %+v", resp.Results[0]) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "7\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, { query: "Distinct(field=likenums)", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{1, 2, 3, 4, 5, 6, 7}) { t.Errorf("wrong values: %+v %+v", resp.Results[0].(*pilosa.Row).Columns(), resp.Results[0].(*pilosa.Row)) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "1\n2\n3\n4\n5\n6\n7\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, { query: "Count(Distinct(field=likes))", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if resp.Results[0].(uint64) != 7 { t.Errorf("wrong count: %+v", resp.Results[0]) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "7\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, { query: "Distinct(field=affinity)", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Pos.Columns(), []uint64{0, 5, 10}) { t.Errorf("wrong positive records: %+v", resp.Results[0].(pilosa.SignedRow).Pos.Columns()) } @@ -6842,10 +6867,16 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong negative records: %+v", resp.Results[0].(pilosa.SignedRow).Neg.Columns()) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "-10\n-5\n0\n5\n10\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, { query: "Distinct(Row(affinity>=0),field=affinity)", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Pos.Columns(), []uint64{0, 5, 10}) { t.Errorf("wrong positive records: %+v", resp.Results[0].(pilosa.SignedRow).Pos.Columns()) } @@ -6853,14 +6884,26 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong negative records: %+v", resp.Results[0].(pilosa.SignedRow).Neg.Columns()) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "0\n5\n10\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, { query: "Count(Distinct(Row(affinity>=0),field=affinity))", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if resp.Results[0].(uint64) != 3 { t.Errorf("wrong number of values: %+v", resp.Results[0]) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "3\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, // Handling this case properly will require changing the way @@ -6879,58 +6922,103 @@ func TestVariousQueries(t *testing.T) { // }, { query: "Distinct(Row(affinity<0),field=likes)", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"pilosa", "zebra", "icecream"}) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "pilosa\nzebra\nicecream\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, { query: "Distinct(Row(affinity>0),field=likes)", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pangolin", "icecream"}) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "molecula\npangolin\nicecream\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, { query: "Distinct(Row(likenums=1),field=likes)", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "icecream"}) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "molecula\nicecream\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, { query: "Distinct(field=likes)", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pilosa", "pangolin", "zebra", "toucan", "dog", "icecream"}) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, { query: "Distinct(All(),field=likes)", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pilosa", "pangolin", "zebra", "toucan", "dog", "icecream"}) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, { query: "Distinct(field=likes )", - verifier: func(t *testing.T, resp pilosa.QueryResponse) { + qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pilosa", "pangolin", "zebra", "toucan", "dog", "icecream"}) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, + csvVerifier: func(t *testing.T, resp string) { + exp := "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n" + if resp != exp { + t.Errorf("expected '%s', got '%s'", exp, resp) + } + }, }, } for i, tst := range tests { t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) { resp := c.Query(t, "users", tst.query) - tst.verifier(t, resp) + tr := c.QueryGRPC(t, "users", tst.query) + if tst.qrVerifier != nil { + tst.qrVerifier(t, resp) + } + csvString := tr.ToCSVString() + // verify everything after header + tst.csvVerifier(t, csvString[strings.Index(csvString, "\n")+1:]) + + // TODO: add HTTP and Postgres and ability to convert + // those results to CSV to run through CSV verifier }) } } diff --git a/proto/interface.go b/proto/interface.go index 77ba3fd8d..321687e94 100644 --- a/proto/interface.go +++ b/proto/interface.go @@ -15,6 +15,8 @@ package proto import ( + "bytes" + "encoding/csv" "fmt" "io" "strings" @@ -325,3 +327,51 @@ func (c ConstRowser) ToRows(fn func(*RowResponse) error) error { return nil } + +func (m *TableResponse) ToCSV(w io.Writer) error { + writer := csv.NewWriter(w) + record := make([]string, len(m.Headers)) + for i, h := range m.Headers { + record[i] = h.Name + } + err := writer.Write(record) + if err != nil { + return errors.Wrap(err, "writing header") + } + for i, row := range m.Rows { + record = record[:0] + for colIndex, col := range row.Columns { + switch m.Headers[colIndex].Datatype { + case "[]string": + record = append(record, fmt.Sprintf("%v", col.GetStringArrayVal())) + case "[]uint64": + record = append(record, fmt.Sprintf("%v", col.GetUint64ArrayVal())) + case "string": + record = append(record, fmt.Sprintf("%v", col.GetStringVal())) + case "uint64": + record = append(record, fmt.Sprintf("%v", col.GetUint64Val())) + case "decimal": + record = append(record, fmt.Sprintf("%v", col.GetDecimalVal().String())) + case "bool": + record = append(record, fmt.Sprintf("%v", col.GetBoolVal())) + case "int64": + record = append(record, fmt.Sprintf("%v", col.GetInt64Val())) + } + } + err := writer.Write(record) + if err != nil { + return errors.Wrapf(err, "writing row %d", i) + } + } + writer.Flush() + return nil +} + +func (m *TableResponse) ToCSVString() string { + buf := &bytes.Buffer{} + err := m.ToCSV(buf) + if err != nil { + panic(fmt.Sprintf("shouldn't get an error writing to bytes.Buffer, got: %v", err)) + } + return buf.String() +} diff --git a/server.go b/server.go index b664e4c87..213d738d1 100644 --- a/server.go +++ b/server.go @@ -499,6 +499,10 @@ func (s *Server) InternalClient() InternalClient { return s.defaultClient } +func (s *Server) GRPCURI() URI { + return s.grpcURI +} + // UpAndDown brings the server up minimally and shuts it down // again; basically, it exists for testing holder open and close. func (s *Server) UpAndDown() error { diff --git a/test/cluster.go b/test/cluster.go index 74b1e31ff..bee281e58 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -26,6 +26,8 @@ import ( "time" "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/api/client" + "github.com/pilosa/pilosa/v2/proto" "github.com/pilosa/pilosa/v2/server" "github.com/pkg/errors" ) @@ -53,6 +55,33 @@ func (c *Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse return c.Nodes[0].QueryAPI(t, &pilosa.QueryRequest{Index: index, Query: query}) } +func (c *Cluster) QueryHTTP(t testing.TB, index, query string) (string, error) { + t.Helper() + if len(c.Nodes) == 0 { + t.Fatal("must have at least one node in cluster to QueryHTTP") + } + return c.Nodes[0].Query(t, index, "", query) +} + +func (c *Cluster) QueryGRPC(t testing.TB, index, query string) *proto.TableResponse { + t.Helper() + if len(c.Nodes) == 0 { + t.Fatal("must have at least one node in cluster to QueryGRPC") + } + + grpcClient, err := client.NewGRPCClient([]string{fmt.Sprintf("%s:%d", c.Nodes[0].Server.GRPCURI().Host, c.Nodes[0].Server.GRPCURI().Port)}, nil) + if err != nil { + t.Fatalf("getting GRPC client: %v", err) + } + + tableResp, err := grpcClient.QueryUnary(context.Background(), index, query) + if err != nil { + t.Fatalf("querying unary: %v", err) + } + + return tableResp +} + func (c *Cluster) GetNode(n int) *Command { return c.Nodes[n] } diff --git a/test/pilosa.go b/test/pilosa.go index d7fdf7aaa..264e3f4c9 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -196,7 +196,7 @@ func (m *Command) Client() *http.InternalClient { } // Query executes a query against the program through the HTTP API. -func (m *Command) Query(t *testing.T, index, rawQuery, query string) (string, error) { +func (m *Command) Query(t testing.TB, index, rawQuery, query string) (string, error) { resp := Do(t, "POST", fmt.Sprintf("%s/index/%s/query?%s", m.URL(), index, rawQuery), query) if resp.StatusCode != gohttp.StatusOK { return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) @@ -285,7 +285,7 @@ func (m *Command) RecalculateCaches(t *testing.T) error { } // Do executes http.Do() with an http.NewRequest(). -func Do(t *testing.T, method, urlStr string, body string) *httpResponse { +func Do(t testing.TB, method, urlStr string, body string) *httpResponse { t.Helper() req, err := gohttp.NewRequest( method, From 27ca9dab369e42349a34e23cbb39ef13441c4938 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 28 Dec 2020 11:14:04 -0600 Subject: [PATCH 08/13] don't hide error getting sign bitmap --- executor.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 190ce07f3..01db6daff 100644 --- a/executor.go +++ b/executor.go @@ -1552,8 +1552,7 @@ func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldNam signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*1, ShardWidth*2) if err != nil { - // TODO wtf... if there's any error getting the sign bitmap we just return an empty result and move on? - return result, nil + return result, errors.Wrap(err, "getting sign bitmap") } dataBitmaps := make([]*roaring.Bitmap, depth) From 757c8a86b5adf755a455b9de19bb1d94533bab6a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 28 Dec 2020 15:22:31 -0600 Subject: [PATCH 09/13] add comments, simplify tests, move ToCSV code generally, address code review feedback --- executor.go | 32 +++++--- executor_test.go | 192 ++++++++++++++++++++++++++------------------- proto/interface.go | 50 ------------ test/cluster.go | 8 +- 4 files changed, 138 insertions(+), 144 deletions(-) diff --git a/executor.go b/executor.go index 01db6daff..cb0f76a06 100644 --- a/executor.go +++ b/executor.go @@ -563,7 +563,6 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q } else { v, err = e.executeCall(ctx, qcx, index, call, shards, opt) } - if err != nil { return nil, err } @@ -6028,13 +6027,26 @@ func (e *executor) translateResults(ctx context.Context, index string, idx *Inde return nil } +// translationStrategy denotes the several different ways the bits in +// a *Row could be translated to string keys. type translationStrategy int const ( + // byCurrentIndex means to interpret the bits as IDs in "top + // level" index for this query (e.g. the index specified in the + // path of the HTTP request). byCurrentIndex translationStrategy = iota + 1 + // byRowField means that the bits in this *Row are row IDs which + // should be translated using the field's (*Row.Field) translation store. byRowField + // byRowFieldForeignIndex means that the bits in this *Row should + // be interpreted as IDs in the foreign index of the *Row.Field. byRowFieldForeignIndex + // byRowIndex means the bits in this *Row should be translated + // according to the index named by *Row.Index byRowIndex + // noTranslation means the bits should not be translated to string + // keys. noTranslation ) @@ -6063,7 +6075,7 @@ func (e *executor) howToTranslate(idx *Index, row *Row) (rowIdx *Index, rowField // Handle the case where the Row has specified a field. if rowField != nil { - // Handle case where field has a foreign index. + // Handle the case where field has a foreign index. if rowField.ForeignIndex() != "" { fidx := e.Holder.Index(rowField.ForeignIndex()) if fidx == nil { @@ -6084,7 +6096,7 @@ func (e *executor) howToTranslate(idx *Index, row *Row) (rowIdx *Index, rowField return rowIdx, rowField, byRowIndex, nil } - // Handle normal case (row represents a set of records in + // Handle the normal case (row represents a set of records in // the top level index, Row has not specifed a different index // or field). if rowIdx == idx && idx.Keys() && rowField == nil { @@ -6096,20 +6108,18 @@ func (e *executor) howToTranslate(idx *Index, row *Row) (rowIdx *Index, rowField func (e *executor) collectResultIDs(index string, idx *Index, call *pql.Call, result interface{}, idSet map[uint64]struct{}) error { switch result := result.(type) { case *Row: + // Only collect result IDs if they are in the current index. _, _, strategy, err := e.howToTranslate(idx, result) if err != nil { return errors.Wrap(err, "determining how to translate") } - // Only collect result IDs if they are in the current index. - if strategy != byCurrentIndex { - return nil - } - for _, segment := range result.Segments() { - for _, col := range segment.Columns() { - idSet[col] = struct{}{} + if strategy == byCurrentIndex { + for _, segment := range result.Segments() { + for _, col := range segment.Columns() { + idSet[col] = struct{}{} + } } } - case ExtractedIDMatrix: for _, col := range result.Columns { idSet[col.ColumnID] = struct{}{} diff --git a/executor_test.go b/executor_test.go index ce51d514e..87af5b276 100644 --- a/executor_test.go +++ b/executor_test.go @@ -17,9 +17,11 @@ package pilosa_test import ( "bytes" "context" + "encoding/csv" "encoding/json" "flag" "fmt" + "io" "io/ioutil" "math" "math/rand" @@ -37,6 +39,7 @@ import ( "github.com/pilosa/pilosa/v2/boltdb" "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/pql" + "github.com/pilosa/pilosa/v2/proto" "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/test" "github.com/pilosa/pilosa/v2/testhook" @@ -6799,7 +6802,7 @@ func TestVariousQueries(t *testing.T) { tests := []struct { query string qrVerifier func(t *testing.T, resp pilosa.QueryResponse) - csvVerifier func(t *testing.T, resp string) + csvVerifier string }{ { query: "Count(All())", @@ -6808,12 +6811,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("expected 6, got %+v", resp.Results[0]) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "6\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "6\n", }, { query: "Count(Distinct(field=likenums))", @@ -6822,12 +6820,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong count: %+v", resp.Results[0]) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "7\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "7\n", }, { query: "Distinct(field=likenums)", @@ -6836,12 +6829,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong values: %+v %+v", resp.Results[0].(*pilosa.Row).Columns(), resp.Results[0].(*pilosa.Row)) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "1\n2\n3\n4\n5\n6\n7\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "1\n2\n3\n4\n5\n6\n7\n", }, { query: "Count(Distinct(field=likes))", @@ -6850,12 +6838,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong count: %+v", resp.Results[0]) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "7\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "7\n", }, { query: "Distinct(field=affinity)", @@ -6867,12 +6850,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong negative records: %+v", resp.Results[0].(pilosa.SignedRow).Neg.Columns()) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "-10\n-5\n0\n5\n10\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "-10\n-5\n0\n5\n10\n", }, { query: "Distinct(Row(affinity>=0),field=affinity)", @@ -6884,12 +6862,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong negative records: %+v", resp.Results[0].(pilosa.SignedRow).Neg.Columns()) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "0\n5\n10\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "0\n5\n10\n", }, { query: "Count(Distinct(Row(affinity>=0),field=affinity))", @@ -6898,12 +6871,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong number of values: %+v", resp.Results[0]) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "3\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "3\n", }, // Handling this case properly will require changing the way @@ -6927,12 +6895,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "pilosa\nzebra\nicecream\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "pilosa\nzebra\nicecream\n", }, { query: "Distinct(Row(affinity>0),field=likes)", @@ -6941,12 +6904,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "molecula\npangolin\nicecream\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "molecula\npangolin\nicecream\n", }, { query: "Distinct(Row(likenums=1),field=likes)", @@ -6955,12 +6913,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "molecula\nicecream\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "molecula\nicecream\n", }, { query: "Distinct(field=likes)", @@ -6969,12 +6922,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n", }, { query: "Distinct(All(),field=likes)", @@ -6983,12 +6931,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n", }, { query: "Distinct(field=likes )", @@ -6997,12 +6940,7 @@ func TestVariousQueries(t *testing.T) { t.Errorf("wrong values: %+v", resp.Results[0]) } }, - csvVerifier: func(t *testing.T, resp string) { - exp := "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n" - if resp != exp { - t.Errorf("expected '%s', got '%s'", exp, resp) - } - }, + csvVerifier: "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n", }, } @@ -7013,12 +6951,104 @@ func TestVariousQueries(t *testing.T) { if tst.qrVerifier != nil { tst.qrVerifier(t, resp) } - csvString := tr.ToCSVString() + csvString, err := tableResponseToCSVString(tr) + if err != nil { + t.Fatal(err) + } // verify everything after header - tst.csvVerifier(t, csvString[strings.Index(csvString, "\n")+1:]) + got := csvString[strings.Index(csvString, "\n")+1:] + if got != tst.csvVerifier { + t.Errorf("expected '%s', got '%s'", tst.csvVerifier, got) + } // TODO: add HTTP and Postgres and ability to convert // those results to CSV to run through CSV verifier }) } } + +func TestReproDistinctWFilterIssue(t *testing.T) { + + c := test.MustRunCluster(t, 3) + defer c.Close() + r := rand.New(rand.NewSource(127)) + + for i := 0; i < 77; i++ { + index := fmt.Sprintf("users%d", i) + c.CreateField(t, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys()) + c.CreateField(t, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "filterfield", pilosa.OptFieldKeys()) + likes := make([][2]string, 0) + filter := make([][2]string, 0) + for userNum := 0; userNum < 100; userNum++ { + likes = append(likes, [2]string{fmt.Sprintf("like%d", r.Intn(95)), "user" + strconv.Itoa(userNum)}) + if r.Intn(10) < 8 { + filter = append(filter, [2]string{"yes", "user" + strconv.Itoa(userNum)}) + } + } + c.ImportKeyKey(t, index, "likes", likes) + c.ImportKeyKey(t, index, "filterfield", filter) + + distinctRes := c.Query(t, index, "Count(Distinct(Row(filterfield=yes), field=likes))") + distinctCount := distinctRes.Results[0].(uint64) + groupbyRes := c.Query(t, index, "GroupBy(Rows(field=likes), filter=Row(filterfield=yes))") + groupbyCount := uint64(len(groupbyRes.Results[0].([]pilosa.GroupCount))) + + t.Logf("D:%v", distinctRes.Results[0]) + t.Logf("G:%v", groupbyRes.Results[0]) + if distinctCount != groupbyCount { + t.Errorf("distinct: %d, groupby: %d", distinctCount, groupbyCount) + } + } +} + +// tableResponseToCSV converts a generic TableResponse to a CSV format +// and writes it to the writer. +func tableResponseToCSV(m *proto.TableResponse, w io.Writer) error { + writer := csv.NewWriter(w) + record := make([]string, len(m.Headers)) + for i, h := range m.Headers { + record[i] = h.Name + } + err := writer.Write(record) + if err != nil { + return errors.Wrap(err, "writing header") + } + for i, row := range m.Rows { + record = record[:0] + for colIndex, col := range row.Columns { + switch m.Headers[colIndex].Datatype { + case "[]string": + record = append(record, fmt.Sprintf("%v", col.GetStringArrayVal())) + case "[]uint64": + record = append(record, fmt.Sprintf("%v", col.GetUint64ArrayVal())) + case "string": + record = append(record, fmt.Sprintf("%v", col.GetStringVal())) + case "uint64": + record = append(record, fmt.Sprintf("%v", col.GetUint64Val())) + case "decimal": + record = append(record, fmt.Sprintf("%v", col.GetDecimalVal().String())) + case "bool": + record = append(record, fmt.Sprintf("%v", col.GetBoolVal())) + case "int64": + record = append(record, fmt.Sprintf("%v", col.GetInt64Val())) + } + } + err := writer.Write(record) + if err != nil { + return errors.Wrapf(err, "writing row %d", i) + } + } + writer.Flush() + return errors.Wrap(writer.Error(), "writing or flushing CSV") +} + +// tableResponseToCSVString converts a generic TableResponse to a CSV format +// and returns it as a string. +func tableResponseToCSVString(m *proto.TableResponse) (string, error) { + buf := &bytes.Buffer{} + err := tableResponseToCSV(m, buf) + if err != nil { + return "", errors.Wrap(err, "writing tableResponse CSV to bytes.Buffer") + } + return buf.String(), nil +} diff --git a/proto/interface.go b/proto/interface.go index 321687e94..77ba3fd8d 100644 --- a/proto/interface.go +++ b/proto/interface.go @@ -15,8 +15,6 @@ package proto import ( - "bytes" - "encoding/csv" "fmt" "io" "strings" @@ -327,51 +325,3 @@ func (c ConstRowser) ToRows(fn func(*RowResponse) error) error { return nil } - -func (m *TableResponse) ToCSV(w io.Writer) error { - writer := csv.NewWriter(w) - record := make([]string, len(m.Headers)) - for i, h := range m.Headers { - record[i] = h.Name - } - err := writer.Write(record) - if err != nil { - return errors.Wrap(err, "writing header") - } - for i, row := range m.Rows { - record = record[:0] - for colIndex, col := range row.Columns { - switch m.Headers[colIndex].Datatype { - case "[]string": - record = append(record, fmt.Sprintf("%v", col.GetStringArrayVal())) - case "[]uint64": - record = append(record, fmt.Sprintf("%v", col.GetUint64ArrayVal())) - case "string": - record = append(record, fmt.Sprintf("%v", col.GetStringVal())) - case "uint64": - record = append(record, fmt.Sprintf("%v", col.GetUint64Val())) - case "decimal": - record = append(record, fmt.Sprintf("%v", col.GetDecimalVal().String())) - case "bool": - record = append(record, fmt.Sprintf("%v", col.GetBoolVal())) - case "int64": - record = append(record, fmt.Sprintf("%v", col.GetInt64Val())) - } - } - err := writer.Write(record) - if err != nil { - return errors.Wrapf(err, "writing row %d", i) - } - } - writer.Flush() - return nil -} - -func (m *TableResponse) ToCSVString() string { - buf := &bytes.Buffer{} - err := m.ToCSV(buf) - if err != nil { - panic(fmt.Sprintf("shouldn't get an error writing to bytes.Buffer, got: %v", err)) - } - return buf.String() -} diff --git a/test/cluster.go b/test/cluster.go index bee281e58..4fd7372d7 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -55,6 +55,9 @@ func (c *Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse return c.Nodes[0].QueryAPI(t, &pilosa.QueryRequest{Index: index, Query: query}) } +// QueryHTTP executes a PQL query through the HTTP endpoint. It fails +// the test for explicit errors, but returns an error which has the +// response body if the HTTP call returns a non-OK status. func (c *Cluster) QueryHTTP(t testing.TB, index, query string) (string, error) { t.Helper() if len(c.Nodes) == 0 { @@ -63,6 +66,8 @@ func (c *Cluster) QueryHTTP(t testing.TB, index, query string) (string, error) { return c.Nodes[0].Query(t, index, "", query) } +// QueryGRPC executes a PQL query through the GRPC endpoint. It fails the +// test if there is an error. func (c *Cluster) QueryGRPC(t testing.TB, index, query string) *proto.TableResponse { t.Helper() if len(c.Nodes) == 0 { @@ -190,8 +195,7 @@ type KeyID struct { ID uint64 } -// ImportIDKey imports data into an index where the index is using -// keys, but the field is not. +//ImportIDKey imports data into an unkeyed set field in a keyed index. func (c *Cluster) ImportIDKey(t testing.TB, index, field string, pairs []KeyID) { t.Helper() importRequest := &pilosa.ImportRequest{ From 21c67352afbff5bada56f6257487e30fb8439d13 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 28 Dec 2020 15:25:13 -0600 Subject: [PATCH 10/13] remove paranoia mode in top level Pilosa --- Makefile | 2 +- nop_paranoia.go | 19 ------------------- paranoia.go | 19 ------------------- row.go | 5 ----- 4 files changed, 1 insertion(+), 44 deletions(-) delete mode 100644 nop_paranoia.go delete mode 100644 paranoia.go diff --git a/Makefile b/Makefile index d622a235c..abd07304d 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) BUILD_TAGS += $(if $(RELEASE_ENABLED),release) BUILD_TAGS += shardwidth$(SHARD_WIDTH) -TEST_TAGS = roaringparanoia paranoia +TEST_TAGS = roaringparanoia define LICENSE_HASH_CODE head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " " endef diff --git a/nop_paranoia.go b/nop_paranoia.go deleted file mode 100644 index 77cdae411..000000000 --- a/nop_paranoia.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !paranoia - -package pilosa - -const paranoia = false diff --git a/paranoia.go b/paranoia.go deleted file mode 100644 index fb717db91..000000000 --- a/paranoia.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build paranoia - -package pilosa - -const paranoia = true diff --git a/row.go b/row.go index 98f5c9275..46a0ec478 100644 --- a/row.go +++ b/row.go @@ -301,11 +301,6 @@ func (r *Row) Union(others ...*Row) *Row { toProcess := make([]*rowSegment, 0, len(others)+1) var output []rowSegment for _, other := range others { - if paranoia { // nolint:staticcheck - // TODO I think there is a good check we can do here, but - // it's nontrivial to check whether two rows are - // compatible because of foreign indexes and such. - } if len(other.segments) > 0 { segments = append(segments, other.segments) } From a3b07ff519a7f97c47983952b46e7b9f7f8750b0 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 28 Dec 2020 15:30:38 -0600 Subject: [PATCH 11/13] re-add log line which has more utility than I thought From Nia: While debugging the Q2 bugs this was somewhat useful in analyzing cluster events. As for the spammy part. . . that seems to be more of an issue with spamming our resets than an issue with the log itself. --- server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server.go b/server.go index 213d738d1..2fb74e560 100644 --- a/server.go +++ b/server.go @@ -661,6 +661,7 @@ func (s *Server) monitorResetTranslationSync() { case <-s.closing: return case <-s.resetTranslationSyncCh: + s.logger.Printf("holder translation sync beginning") s.wg.Add(1) go func() { // Obtaining this lock ensures that there is only From 790bea147f8e769e6cff1dc38266b59b3576ebb2 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 28 Dec 2020 15:59:49 -0600 Subject: [PATCH 12/13] make view and fragment not found errors constant based on code review feedback --- executor.go | 3 ++- rrtx.go | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/executor.go b/executor.go index cb0f76a06..1910d42e8 100644 --- a/executor.go +++ b/executor.go @@ -1537,7 +1537,8 @@ func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldNam existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*0, ShardWidth*1) if err != nil { - if _, ok := errors.Cause(err).(ViewOrFragmentNotFound); ok { + switch errors.Cause(err) { + case ViewNotFound, FragmentNotFound: return result, nil } return result, errors.Wrap(err, "getting exists bitmap") diff --git a/rrtx.go b/rrtx.go index 0b4d7d1e7..6113067c4 100644 --- a/rrtx.go +++ b/rrtx.go @@ -373,13 +373,13 @@ func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*frag v := f.view(view) if v == nil { - return nil, ViewOrFragmentNotFound(errors.Errorf("view not found: %q", view)) + return nil, errors.Wrapf(ViewNotFound, "getting %s", view) } frag := v.Fragment(shard) if frag == nil { - return nil, ViewOrFragmentNotFound(errors.Errorf("fragment not found: %q / %q / %d", field, view, shard)) + return nil, errors.Wrapf(FragmentNotFound, "field:%q, view:%q, shard:%d", field, view, shard) } // Note: we cannot cache frag into tx.fragment. @@ -389,7 +389,8 @@ func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*frag return frag, nil } -type ViewOrFragmentNotFound error +const ViewNotFound = Error("view not found") +const FragmentNotFound = Error("fragment not found") func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { frag, err := tx.getFragment(index, field, view, shard) From 1ed91ebad3f2f54063558ffac475bf1038b08292 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 28 Dec 2020 16:01:07 -0600 Subject: [PATCH 13/13] simplify error messages also remove test which was accidentally committed --- executor_test.go | 34 ---------------------------------- test/cluster.go | 5 +++-- 2 files changed, 3 insertions(+), 36 deletions(-) diff --git a/executor_test.go b/executor_test.go index 87af5b276..a20d60495 100644 --- a/executor_test.go +++ b/executor_test.go @@ -6967,40 +6967,6 @@ func TestVariousQueries(t *testing.T) { } } -func TestReproDistinctWFilterIssue(t *testing.T) { - - c := test.MustRunCluster(t, 3) - defer c.Close() - r := rand.New(rand.NewSource(127)) - - for i := 0; i < 77; i++ { - index := fmt.Sprintf("users%d", i) - c.CreateField(t, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys()) - c.CreateField(t, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "filterfield", pilosa.OptFieldKeys()) - likes := make([][2]string, 0) - filter := make([][2]string, 0) - for userNum := 0; userNum < 100; userNum++ { - likes = append(likes, [2]string{fmt.Sprintf("like%d", r.Intn(95)), "user" + strconv.Itoa(userNum)}) - if r.Intn(10) < 8 { - filter = append(filter, [2]string{"yes", "user" + strconv.Itoa(userNum)}) - } - } - c.ImportKeyKey(t, index, "likes", likes) - c.ImportKeyKey(t, index, "filterfield", filter) - - distinctRes := c.Query(t, index, "Count(Distinct(Row(filterfield=yes), field=likes))") - distinctCount := distinctRes.Results[0].(uint64) - groupbyRes := c.Query(t, index, "GroupBy(Rows(field=likes), filter=Row(filterfield=yes))") - groupbyCount := uint64(len(groupbyRes.Results[0].([]pilosa.GroupCount))) - - t.Logf("D:%v", distinctRes.Results[0]) - t.Logf("G:%v", groupbyRes.Results[0]) - if distinctCount != groupbyCount { - t.Errorf("distinct: %d, groupby: %d", distinctCount, groupbyCount) - } - } -} - // tableResponseToCSV converts a generic TableResponse to a CSV format // and writes it to the writer. func tableResponseToCSV(m *proto.TableResponse, w io.Writer) error { diff --git a/test/cluster.go b/test/cluster.go index 4fd7372d7..6bdfc8153 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -61,8 +61,9 @@ func (c *Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse func (c *Cluster) QueryHTTP(t testing.TB, index, query string) (string, error) { t.Helper() if len(c.Nodes) == 0 { - t.Fatal("must have at least one node in cluster to QueryHTTP") + t.Fatal("must have at least one node in cluster to query") } + return c.Nodes[0].Query(t, index, "", query) } @@ -71,7 +72,7 @@ func (c *Cluster) QueryHTTP(t testing.TB, index, query string) (string, error) { func (c *Cluster) QueryGRPC(t testing.TB, index, query string) *proto.TableResponse { t.Helper() if len(c.Nodes) == 0 { - t.Fatal("must have at least one node in cluster to QueryGRPC") + t.Fatal("must have at least one node in cluster to query") } grpcClient, err := client.NewGRPCClient([]string{fmt.Sprintf("%s:%d", c.Nodes[0].Server.GRPCURI().Host, c.Nodes[0].Server.GRPCURI().Port)}, nil)