scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-05 08:10:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1261 from jaffee/some-distinct-bugs

Browse source 
fix Count(Distinct) bug and add better tests
This commit is contained in:
Matthew Jaffee 2020-12-28 15:24:50 -07:00 • committed by GitHub
commit 4447d6fa76
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 872 additions and 187 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
encoding/proto/proto.go
View file

@ -1453,6 +1453,8 @@ func (s Serializer) decodeRow(pr *internal.Row) *pilosa.Row {
}
r.Attrs = s.decodeAttrs(pr.Attrs)
r.Keys = pr.Keys
r.Index = pr.Index
r.Field = pr.Field
return r
}
@ -1700,6 +1702,8 @@ func (s Serializer) encodeRow(r *pilosa.Row) *internal.Row {
ir := &internal.Row{
Keys: r.Keys,
Attrs: s.encodeAttrs(r.Attrs),
Index: r.Index,
Field: r.Field,
}
if s.RoaringRows {
ir.Roaring = r.Roaring()

309
executor.go
View file

@ -361,7 +361,7 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr
}
// handlePreCalls traverses the call tree looking for calls that need
// precomputed values. Right now, that's just Distinct.
// precomputed values (e.g. Distinct, UnionRows, ConstRow...).
func (e *executor) handlePreCalls(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
if c.Name == "Precomputed" {
idx := c.Args["valueidx"].(int64)
@ -1080,8 +1080,9 @@ func (e *executor) executeSum(ctx context.Context, qcx *Qcx, index string, c *pq
return other, nil
}
// executeDistinct executes a Distinct call on a field.
func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (SignedRow, error) {
// executeDistinct executes a Distinct call on a field. It returns a
// SignedRow for int fields and a *Row for set/mutex/time fields.
func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinct")
defer span.Finish()
@ -1099,21 +1100,31 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string,
// Merge returned results at coordinating node.
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(SignedRow)
if err := ctx.Err(); err != nil {
return err
}
return other.union(v.(SignedRow))
switch other := prev.(type) {
case SignedRow:
return other.union(v.(SignedRow))
case *Row:
return other.Union(v.(*Row))
case nil:
return v
default:
return errors.Errorf("unexpected return type from executeDistinctShard: %+v %T", other, other)
}
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return SignedRow{}, err
return nil, err
}
other, _ := result.(SignedRow)
other.field = field
return other, nil
if other, ok := result.(SignedRow); ok {
other.field = field
}
return result, nil
}
// executeMin executes a Min() call.
@ -1302,7 +1313,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, qcx *Qcx, index string
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(*Row)
if other == nil {
// TODO... what's going on on the following line
other = NewRow() // bug! this row ends up containing Badger Txn data that should be accessed outside the Txn.
}
if err := ctx.Err(); err != nil {
@ -1398,14 +1409,23 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, qcx *Qcx, index s
// executeDistinctShard executes a Distinct call on a single shard, yielding
// a SignedRow of the values found.
func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index string, fieldName string, c *pql.Call, shard uint64) (result SignedRow, err error) {
func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index string, fieldName string, c *pql.Call, shard uint64) (result interface{}, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinctShard")
defer span.Finish()
idx := e.Holder.Index(index)
field := e.Holder.Field(index, fieldName)
if field == nil {
return SignedRow{}, ErrFieldNotFound
return nil, ErrFieldNotFound
}
bsig := field.bsiGroup(fieldName)
if bsig == nil {
result = &Row{
Index: index,
Field: fieldName,
}
} else {
result = SignedRow{}
}
var filter *Row
@ -1427,28 +1447,27 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str
// can go ahead and save time by returning the empty results,
// because the filter excluded everything.
if filterBitmap == nil || !filterBitmap.Any() {
return SignedRow{}, nil
return result, nil
}
}
bsig := field.bsiGroup(fieldName)
if bsig == nil {
return executeDistinctShardSet(ctx, qcx, idx, fieldName, shard, filterBitmap)
}
return executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap)
}
func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result SignedRow, err0 error) {
func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result *Row, err0 error) {
index := idx.Name()
tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard})
if err != nil {
return SignedRow{}, err
return nil, err
}
defer finisher(&err0)
fragData, _, err := tx.ContainerIterator(index, fieldName, "standard", shard, 0)
if err != nil {
return SignedRow{}, errors.Wrap(err, "getting fragment data")
return nil, errors.Wrap(err, "getting fragment data")
}
defer fragData.Close()
// We can't grab the containers "for each row" from the set-type field,
@ -1486,20 +1505,22 @@ func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldNam
if roaring.IntersectionAny(c, filter[k%(1<<shardVsContainerExponent)]) {
_, err = rows.Add(row)
if err != nil {
return SignedRow{}, errors.Wrap(err, "collecting results")
return nil, errors.Wrap(err, "collecting results")
}
seenThisRow = true
}
} else if c.N() != 0 {
_, err = rows.Add(row)
if err != nil {
return SignedRow{}, errors.Wrap(err, "recording results")
return nil, errors.Wrap(err, "recording results")
}
seenThisRow = true
}
}
return SignedRow{Pos: NewRowFromBitmap(rows)}, nil
result = NewRowFromBitmap(rows)
result.Index = idx.Name()
result.Field = fieldName
return result, nil
}
func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, bsig *bsiGroup, filterBitmap *roaring.Bitmap) (result SignedRow, err0 error) {
@ -1516,7 +1537,11 @@ func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldNam
existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*0, ShardWidth*1)
if err != nil {
return result, err
switch errors.Cause(err) {
case ViewNotFound, FragmentNotFound:
return result, nil
}
return result, errors.Wrap(err, "getting exists bitmap")
}
if filterBitmap != nil {
existsBitmap = existsBitmap.Intersect(filterBitmap)
@ -1527,7 +1552,7 @@ func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldNam
signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*1, ShardWidth*2)
if err != nil {
return result, nil
return result, errors.Wrap(err, "getting sign bitmap")
}
dataBitmaps := make([]*roaring.Bitmap, depth)
@ -1608,10 +1633,14 @@ func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldNam
}
}
}
return SignedRow{
result = SignedRow{
Neg: NewRowFromBitmap(negBitmap),
Pos: NewRowFromBitmap(posBitmap),
}, nil
}
result.Neg.Index, result.Pos.Index = idx.Name(), idx.Name()
result.Neg.Field, result.Pos.Field = fieldName, fieldName
return result, nil
}
// executeSumCountShard calculates the sum and count for bsiGroups on a shard.
@ -4235,9 +4264,35 @@ func (e *executor) executeCount(ctx context.Context, qcx *Qcx, index string, c *
return 0, errors.New("Count() only accepts a single bitmap input")
}
child := c.Children[0]
// If the child is precomputed, we'll bypass mapreduce, ignore
// shards, and just count the number of bits.
if child.Name == "Precomputed" {
count := uint64(0)
for _, irow := range child.Precomputed {
switch row := irow.(type) {
case *Row:
for _, seg := range row.segments {
count += seg.n
}
case SignedRow:
for _, seg := range row.Pos.segments {
count += seg.n
}
for _, seg := range row.Neg.segments {
count += seg.n
}
default:
return 0, errors.Errorf("unexpected precomputed value type inside count: %+v", row)
}
}
return count, nil
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(ctx context.Context, shard uint64) (_ interface{}, err error) {
row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
row, err := e.executeBitmapCallShard(ctx, qcx, index, child, shard)
if err != nil {
return 0, err
}
@ -5135,7 +5190,10 @@ func makeEmbeddedDataForShards(allRows []*Row, shards []uint64) []*Row {
}
segments := row.segments
segmentIndex := 0
newRows[i] = &Row{}
newRows[i] = &Row{
Index: row.Index,
Field: row.Field,
}
for _, shard := range shards {
for segmentIndex < len(segments) && segments[segmentIndex].shard < shard {
segmentIndex++
@ -5970,19 +6028,99 @@ func (e *executor) translateResults(ctx context.Context, index string, idx *Inde
return nil
}
// translationStrategy denotes the several different ways the bits in
// a *Row could be translated to string keys.
type translationStrategy int
const (
// byCurrentIndex means to interpret the bits as IDs in "top
// level" index for this query (e.g. the index specified in the
// path of the HTTP request).
byCurrentIndex translationStrategy = iota + 1
// byRowField means that the bits in this *Row are row IDs which
// should be translated using the field's (*Row.Field) translation store.
byRowField
// byRowFieldForeignIndex means that the bits in this *Row should
// be interpreted as IDs in the foreign index of the *Row.Field.
byRowFieldForeignIndex
// byRowIndex means the bits in this *Row should be translated
// according to the index named by *Row.Index
byRowIndex
// noTranslation means the bits should not be translated to string
// keys.
noTranslation
)
// howToTranslate determines how a *Row object's bits should be
// translated to keys (if at all). There are several different options
// detailed by the various const values of translationStrategy. In
// order to do this it has to figure out the row's index and field
// which it also returns as the caller may need them to actually
// execute the translation or do whatever else it's doing with the
// translationStrategy information.
func (e *executor) howToTranslate(idx *Index, row *Row) (rowIdx *Index, rowField *Field, strat translationStrategy, err error) {
// First get the index and field the row specifies (if any).
rowIdx = idx
if row.Index != "" && row.Index != idx.Name() {
rowIdx = e.Holder.Index(row.Index)
if rowIdx == nil {
return nil, nil, 0, errors.Errorf("got a row with unknown index: %s", row.Index)
}
}
if row.Field != "" {
rowField = rowIdx.Field(row.Field)
if rowField == nil {
return nil, nil, 0, errors.Errorf("got a row with unknown index/field %s/%s", idx.Name(), row.Field)
}
}
// Handle the case where the Row has specified a field.
if rowField != nil {
// Handle the case where field has a foreign index.
if rowField.ForeignIndex() != "" {
fidx := e.Holder.Index(rowField.ForeignIndex())
if fidx == nil {
return nil, nil, 0, errors.Errorf("foreign index %s not found for field %s in index %s", rowField.ForeignIndex(), rowField.Name(), rowField.Index())
}
if fidx.Keys() {
return rowIdx, rowField, byRowFieldForeignIndex, nil
}
} else if rowField.Keys() {
return rowIdx, rowField, byRowField, nil
}
return rowIdx, rowField, noTranslation, nil
}
// In this case, the row has specified an index, but not a field,
// so we translate according to that index.
if rowIdx != idx && rowIdx.Keys() {
return rowIdx, rowField, byRowIndex, nil
}
// Handle the normal case (row represents a set of records in
// the top level index, Row has not specifed a different index
// or field).
if rowIdx == idx && idx.Keys() && rowField == nil {
return rowIdx, rowField, byCurrentIndex, nil
}
return rowIdx, rowField, noTranslation, nil
}
func (e *executor) collectResultIDs(index string, idx *Index, call *pql.Call, result interface{}, idSet map[uint64]struct{}) error {
switch result := result.(type) {
case *Row:
if !idx.Keys() {
return nil
// Only collect result IDs if they are in the current index.
_, _, strategy, err := e.howToTranslate(idx, result)
if err != nil {
return errors.Wrap(err, "determining how to translate")
}
for _, segment := range result.Segments() {
for _, col := range segment.Columns() {
idSet[col] = struct{}{}
if strategy == byCurrentIndex {
for _, segment := range result.Segments() {
for _, col := range segment.Columns() {
idSet[col] = struct{}{}
}
}
}
case ExtractedIDMatrix:
for _, col := range result.Columns {
idSet[col.ColumnID] = struct{}{}
@ -6005,10 +6143,14 @@ func (e *executor) preTranslateMatrixSet(mat ExtractedIDMatrix, fieldIdx uint, f
}
func (e *executor) translateResult(ctx context.Context, index string, idx *Index, call *pql.Call, result interface{}, idSet map[uint64]string) (_ interface{}, err error) {
switch result := result.(type) {
case *Row:
if idx.Keys() {
rowIdx, rowField, strategy, err := e.howToTranslate(idx, result)
if err != nil {
return nil, errors.Wrap(err, "determining translation strategy")
}
switch strategy {
case byCurrentIndex:
other := &Row{Attrs: result.Attrs}
for _, segment := range result.Segments() {
for _, col := range segment.Columns() {
@ -6016,11 +6158,40 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
}
}
return other, nil
}
case byRowField:
keys, err := e.Cluster.translateFieldListIDs(rowField, result.Columns())
if err != nil {
return nil, errors.Wrap(err, "translating Row to field keys")
}
result.Keys = keys
case byRowFieldForeignIndex:
idx = e.Holder.Index(rowField.ForeignIndex())
if idx == nil {
return nil, errors.Errorf("foreign index %s not found for field %s in index %s", rowField.ForeignIndex(), rowField.Name(), rowField.Index())
}
for _, segment := range result.Segments() {
keys, err := e.Cluster.translateIndexIDs(context.Background(), rowField.ForeignIndex(), segment.Columns())
if err != nil {
return nil, errors.Wrap(err, "translating index ids")
}
result.Keys = append(result.Keys, keys...)
}
// TODO: instead of supporting SignedRow here, we may be able to
// make the return type for an int field with a ForeignIndex be
// a *Row instead (because it should always be positive).
case byRowIndex:
for _, segment := range result.Segments() {
keys, err := e.Cluster.translateIndexIDs(context.Background(), rowIdx.Name(), segment.Columns())
if err != nil {
return nil, errors.Wrap(err, "translating index ids")
}
result.Keys = append(result.Keys, keys...)
}
return result, nil
case noTranslation:
return result, nil
default:
return nil, errors.Errorf("unknown translation strategy %d", strategy)
}
case SignedRow:
sr, err := func() (*SignedRow, error) {
fieldName := callArgString(call, "field")
@ -6529,38 +6700,42 @@ func (s SignedRow) ToTable() (*pb.TableResponse, error) {
func (s SignedRow) ToRows(callback func(*pb.RowResponse) error) error {
ci := []*pb.ColumnInfo{{Name: s.Field(), Datatype: "int64"}}
negs := s.Neg.Columns()
for i := len(negs) - 1; i >= 0; i-- {
val, err := toNegInt64(negs[i])
if err != nil {
return errors.Wrap(err, "converting uint64 to int64 (negative)")
}
if s.Neg != nil {
negs := s.Neg.Columns()
for i := len(negs) - 1; i >= 0; i-- {
val, err := toNegInt64(negs[i])
if err != nil {
return errors.Wrap(err, "converting uint64 to int64 (negative)")
}
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: val}},
},
}); err != nil {
return errors.Wrap(err, "calling callback")
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: val}},
},
}); err != nil {
return errors.Wrap(err, "calling callback")
}
ci = nil
}
ci = nil
}
for _, id := range s.Pos.Columns() {
val, err := toInt64(id)
if err != nil {
return errors.Wrap(err, "converting uint64 to int64 (positive)")
}
if s.Pos != nil {
for _, id := range s.Pos.Columns() {
val, err := toInt64(id)
if err != nil {
return errors.Wrap(err, "converting uint64 to int64 (positive)")
}
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: val}},
},
}); err != nil {
return errors.Wrap(err, "calling callback")
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: val}},
},
}); err != nil {
return errors.Wrap(err, "calling callback")
}
ci = nil
}
ci = nil
}
return nil
}

294
executor_test.go
View file

@ -17,9 +17,11 @@ package pilosa_test
import (
"bytes"
"context"
"encoding/csv"
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"math"
"math/rand"
@ -37,6 +39,7 @@ import (
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/http"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/proto"
"github.com/pilosa/pilosa/v2/server"
"github.com/pilosa/pilosa/v2/test"
"github.com/pilosa/pilosa/v2/testhook"
@ -5434,9 +5437,9 @@ func TestExecutor_ForeignIndex(t *testing.T) {
if !sameStringSlice(distinct.Pos.Keys, []string{"one", "two", "twenty-one"}) {
t.Fatalf("unexpected keys: %v", distinct.Pos.Keys)
}
distinct = c.Query(t, "child", `Distinct(index="child", field="parent_set_id")`).Results[0].(pilosa.SignedRow)
if !sameStringSlice(distinct.Pos.Keys, []string{"one", "two", "twenty-one"}) {
t.Fatalf("unexpected keys: %v", distinct.Pos.Keys)
row := c.Query(t, "child", `Distinct(index="child", field="parent_set_id")`).Results[0].(*pilosa.Row)
if !sameStringSlice(row.Keys, []string{"one", "two", "twenty-one"}) {
t.Fatalf("unexpected keys: %v", row.Keys)
}
eq := c.Query(t, "child", `Row(parent_id=="one")`).Results[0].(*pilosa.Row)
@ -6512,7 +6515,6 @@ func TestExecutor_BareDistinct(t *testing.T) {
c.CreateField(t, "i", pilosa.IndexOptions{}, "ints",
pilosa.OptFieldTypeInt(0, math.MaxInt64),
)
c.CreateField(t, "i", pilosa.IndexOptions{}, "set")
c.CreateField(t, "i", pilosa.IndexOptions{}, "filter")
// Populate integer data.
@ -6521,17 +6523,13 @@ func TestExecutor_BareDistinct(t *testing.T) {
Set(%d, ints=2)
`, ShardWidth))
c.Query(t, "i", fmt.Sprintf(`
Set(0, set=1)
Set(1, set=2)
Set(%d, set=2)
Set(0, filter=1)
Set(%d, filter=1)
`, 65537, 65537))
`, 65537))
for _, pql := range []string{
`Distinct(field="ints")`,
`Distinct(index="i", field="ints")`,
`Distinct(Row(filter=1), field="set")`,
} {
exp := []uint64{1, 2}
res := c.Query(t, "i", pql).Results[0].(pilosa.SignedRow)
@ -6742,3 +6740,281 @@ func TestMissingKeyRegression(t *testing.T) {
})
}
}
// TestVariousQueries has originally been written to test out a
// variety of scenarios with Distinct, but it's structure is more
// general purpose. My vision is to eventually have any test which
// needs to test a single query be in here, and have a robust enough
// test data set loaded at the start which covers what we want to
// test.
//
// I'd also like to have it automatically run a matrix of scenarios
// (single and multi-node clusters, different endpoints for the
// queries (HTTP, GRPC, Postgres), etc.).
func TestVariousQueries(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
// Create and populate "likenums" similar to "likes", but without keys on the field.
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums")
c.ImportIDKey(t, "users", "likenums", []test.KeyID{
{ID: 1, Key: "userA"},
{ID: 2, Key: "userB"},
{ID: 3, Key: "userC"},
{ID: 4, Key: "userD"},
{ID: 5, Key: "userE"},
{ID: 6, Key: "userF"},
{ID: 7, Key: "userA"},
{ID: 7, Key: "userB"},
{ID: 7, Key: "userC"},
{ID: 7, Key: "userD"},
{ID: 7, Key: "userE"},
{ID: 7, Key: "userF"},
})
// Create and populate "likes" field.
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys())
c.ImportKeyKey(t, "users", "likes", [][2]string{
{"molecula", "userA"},
{"pilosa", "userB"},
{"pangolin", "userC"},
{"zebra", "userD"},
{"toucan", "userE"},
{"dog", "userF"},
{"icecream", "userA"},
{"icecream", "userB"},
{"icecream", "userC"},
{"icecream", "userD"},
{"icecream", "userE"},
{"icecream", "userF"},
})
// Create and populate "affinity" int field with negative, positive, zero and null values.
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "affinity", pilosa.OptFieldTypeInt(-1000, 1000))
c.ImportIntKey(t, "users", "affinity", []test.IntKey{
{Val: 10, Key: "userA"},
{Val: -10, Key: "userB"},
{Val: 5, Key: "userC"},
{Val: -5, Key: "userD"},
{Val: 0, Key: "userE"},
})
tests := []struct {
query string
qrVerifier func(t *testing.T, resp pilosa.QueryResponse)
csvVerifier string
}{
{
query: "Count(All())",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if resp.Results[0].(uint64) != 6 {
t.Errorf("expected 6, got %+v", resp.Results[0])
}
},
csvVerifier: "6\n",
},
{
query: "Count(Distinct(field=likenums))",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if resp.Results[0].(uint64) != 7 {
t.Errorf("wrong count: %+v", resp.Results[0])
}
},
csvVerifier: "7\n",
},
{
query: "Distinct(field=likenums)",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{1, 2, 3, 4, 5, 6, 7}) {
t.Errorf("wrong values: %+v %+v", resp.Results[0].(*pilosa.Row).Columns(), resp.Results[0].(*pilosa.Row))
}
},
csvVerifier: "1\n2\n3\n4\n5\n6\n7\n",
},
{
query: "Count(Distinct(field=likes))",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if resp.Results[0].(uint64) != 7 {
t.Errorf("wrong count: %+v", resp.Results[0])
}
},
csvVerifier: "7\n",
},
{
query: "Distinct(field=affinity)",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Pos.Columns(), []uint64{0, 5, 10}) {
t.Errorf("wrong positive records: %+v", resp.Results[0].(pilosa.SignedRow).Pos.Columns())
}
if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Neg.Columns(), []uint64{5, 10}) {
t.Errorf("wrong negative records: %+v", resp.Results[0].(pilosa.SignedRow).Neg.Columns())
}
},
csvVerifier: "-10\n-5\n0\n5\n10\n",
},
{
query: "Distinct(Row(affinity>=0),field=affinity)",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Pos.Columns(), []uint64{0, 5, 10}) {
t.Errorf("wrong positive records: %+v", resp.Results[0].(pilosa.SignedRow).Pos.Columns())
}
if !reflect.DeepEqual(resp.Results[0].(pilosa.SignedRow).Neg.Columns(), []uint64{}) {
t.Errorf("wrong negative records: %+v", resp.Results[0].(pilosa.SignedRow).Neg.Columns())
}
},
csvVerifier: "0\n5\n10\n",
},
{
query: "Count(Distinct(Row(affinity>=0),field=affinity))",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if resp.Results[0].(uint64) != 3 {
t.Errorf("wrong number of values: %+v", resp.Results[0])
}
},
csvVerifier: "3\n",
},
// Handling this case properly will require changing the way
// that precomputed data is stored on Call objects. Currently
// if a Distinct is at all nested (e.g. within a Count) it
// gets handled by executor.handlePreCalls which assumes that
// only the positive values are worthwhile.
//
// {
// query: "Count(Distinct(field=affinity))",
// verifier: func(t *testing.T, resp pilosa.QueryResponse) {
// if resp.Results[0].(uint64) != 5 {
// t.Errorf("wrong number of values: %+v", resp.Results[0])
// }
// },
// },
{
query: "Distinct(Row(affinity<0),field=likes)",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"pilosa", "zebra", "icecream"}) {
t.Errorf("wrong values: %+v", resp.Results[0])
}
},
csvVerifier: "pilosa\nzebra\nicecream\n",
},
{
query: "Distinct(Row(affinity>0),field=likes)",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pangolin", "icecream"}) {
t.Errorf("wrong values: %+v", resp.Results[0])
}
},
csvVerifier: "molecula\npangolin\nicecream\n",
},
{
query: "Distinct(Row(likenums=1),field=likes)",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "icecream"}) {
t.Errorf("wrong values: %+v", resp.Results[0])
}
},
csvVerifier: "molecula\nicecream\n",
},
{
query: "Distinct(field=likes)",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pilosa", "pangolin", "zebra", "toucan", "dog", "icecream"}) {
t.Errorf("wrong values: %+v", resp.Results[0])
}
},
csvVerifier: "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n",
},
{
query: "Distinct(All(),field=likes)",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pilosa", "pangolin", "zebra", "toucan", "dog", "icecream"}) {
t.Errorf("wrong values: %+v", resp.Results[0])
}
},
csvVerifier: "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n",
},
{
query: "Distinct(field=likes )",
qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) {
if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Keys, []string{"molecula", "pilosa", "pangolin", "zebra", "toucan", "dog", "icecream"}) {
t.Errorf("wrong values: %+v", resp.Results[0])
}
},
csvVerifier: "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n",
},
}
for i, tst := range tests {
t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) {
resp := c.Query(t, "users", tst.query)
tr := c.QueryGRPC(t, "users", tst.query)
if tst.qrVerifier != nil {
tst.qrVerifier(t, resp)
}
csvString, err := tableResponseToCSVString(tr)
if err != nil {
t.Fatal(err)
}
// verify everything after header
got := csvString[strings.Index(csvString, "\n")+1:]
if got != tst.csvVerifier {
t.Errorf("expected '%s', got '%s'", tst.csvVerifier, got)
}
// TODO: add HTTP and Postgres and ability to convert
// those results to CSV to run through CSV verifier
})
}
}
// tableResponseToCSV converts a generic TableResponse to a CSV format
// and writes it to the writer.
func tableResponseToCSV(m *proto.TableResponse, w io.Writer) error {
writer := csv.NewWriter(w)
record := make([]string, len(m.Headers))
for i, h := range m.Headers {
record[i] = h.Name
}
err := writer.Write(record)
if err != nil {
return errors.Wrap(err, "writing header")
}
for i, row := range m.Rows {
record = record[:0]
for colIndex, col := range row.Columns {
switch m.Headers[colIndex].Datatype {
case "[]string":
record = append(record, fmt.Sprintf("%v", col.GetStringArrayVal()))
case "[]uint64":
record = append(record, fmt.Sprintf("%v", col.GetUint64ArrayVal()))
case "string":
record = append(record, fmt.Sprintf("%v", col.GetStringVal()))
case "uint64":
record = append(record, fmt.Sprintf("%v", col.GetUint64Val()))
case "decimal":
record = append(record, fmt.Sprintf("%v", col.GetDecimalVal().String()))
case "bool":
record = append(record, fmt.Sprintf("%v", col.GetBoolVal()))
case "int64":
record = append(record, fmt.Sprintf("%v", col.GetInt64Val()))
}
}
err := writer.Write(record)
if err != nil {
return errors.Wrapf(err, "writing row %d", i)
}
}
writer.Flush()
return errors.Wrap(writer.Error(), "writing or flushing CSV")
}
// tableResponseToCSVString converts a generic TableResponse to a CSV format
// and returns it as a string.
func tableResponseToCSVString(m *proto.TableResponse) (string, error) {
buf := &bytes.Buffer{}
err := tableResponseToCSV(m, buf)
if err != nil {
return "", errors.Wrap(err, "writing tableResponse CSV to bytes.Buffer")
}
return buf.String(), nil
}

314
internal/public.pb.go
View file

@ -28,6 +28,8 @@ type Row struct {
Keys []string `protobuf:"bytes,3,rep,name=Keys,proto3" json:"Keys,omitempty"`
Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,proto3" json:"Attrs,omitempty"`
Roaring []byte `protobuf:"bytes,4,opt,name=Roaring,proto3" json:"Roaring,omitempty"`
Index string `protobuf:"bytes,5,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,6,opt,name=Field,proto3" json:"Field,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -94,6 +96,20 @@ func (m *Row) GetRoaring() []byte {
return nil
}
func (m *Row) GetIndex() string {
if m != nil {
return m.Index
}
return ""
}
func (m *Row) GetField() string {
if m != nil {
return m.Field
}
return ""
}
type RowMatrix struct {
Rows []*Row `protobuf:"bytes,1,rep,name=Rows,proto3" json:"Rows,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -2667,113 +2683,113 @@ func init() {
func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) }
var fileDescriptor_413a91106d7bcce8 = []byte{
// 1682 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0xdb, 0xca,
0x11, 0x37, 0x45, 0xca, 0x92, 0x46, 0xb2, 0xe3, 0xb7, 0xd1, 0x7b, 0x25, 0x52, 0xc7, 0x4f, 0x25,
0xdc, 0x3e, 0xb5, 0x28, 0x1c, 0x38, 0x4d, 0x82, 0x5c, 0xda, 0xc6, 0x8e, 0x9c, 0x9a, 0x48, 0xed,
0xa6, 0x2b, 0xc3, 0xb9, 0x15, 0xa0, 0xa5, 0xad, 0x43, 0x94, 0x12, 0x55, 0x8a, 0x8a, 0xec, 0x4b,
0x81, 0x7e, 0x86, 0x5c, 0xfa, 0x11, 0x7a, 0xea, 0x87, 0xe8, 0xa5, 0x3d, 0xf6, 0x58, 0xa0, 0x97,
0x22, 0xed, 0xb7, 0xe8, 0xa5, 0x98, 0x59, 0x2e, 0x77, 0x49, 0xd1, 0x8e, 0x11, 0xbc, 0xdb, 0xce,
0x9f, 0x9d, 0x9d, 0xf9, 0xcd, 0xec, 0xec, 0x90, 0xd0, 0x99, 0x2d, 0x2e, 0xa2, 0x70, 0xb4, 0x37,
0x4b, 0xe2, 0x34, 0x66, 0xcd, 0x70, 0x9a, 0x8a, 0x64, 0x1a, 0x44, 0xde, 0x1c, 0x6c, 0x1e, 0x2f,
0x99, 0x0b, 0x8d, 0x97, 0x71, 0xb4, 0x98, 0x4c, 0xe7, 0xae, 0xd5, 0xb3, 0xfb, 0x0e, 0x57, 0x24,
0x63, 0xe0, 0xbc, 0x16, 0xd7, 0x73, 0xd7, 0xee, 0xd9, 0xfd, 0x16, 0xa7, 0x35, 0xdb, 0x85, 0xfa,
0x41, 0x9a, 0x26, 0x73, 0xb7, 0xd6, 0xb3, 0xfb, 0xed, 0xc7, 0x9b, 0x7b, 0xca, 0xdc, 0x1e, 0xb2,
0xb9, 0x14, 0xa2, 0x4d, 0x1e, 0x07, 0x49, 0x38, 0xbd, 0x74, 0x9d, 0x9e, 0xd5, 0xef, 0x70, 0x45,
0x7a, 0x7b, 0xd0, 0xe2, 0xf1, 0xf2, 0x24, 0x48, 0x93, 0xf0, 0x8a, 0x7d, 0x0f, 0x1c, 0x1e, 0x2f,
0xe5, 0xb9, 0xed, 0xc7, 0x1b, 0xda, 0x16, 0x8f, 0x97, 0x9c, 0x44, 0xde, 0x09, 0xb4, 0x86, 0xe1,
0xe5, 0x54, 0x8c, 0xd1, 0xd5, 0xaf, 0xc1, 0x7e, 0x13, 0xa3, 0xba, 0xb5, 0xaa, 0x8e, 0x12, 0x54,
0x38, 0x15, 0x97, 0x6e, 0xad, 0x52, 0xe1, 0x54, 0x5c, 0x7a, 0xcf, 0x61, 0x93, 0xc7, 0x4b, 0x7f,
0x2c, 0xa6, 0x69, 0xf8, 0xdb, 0x50, 0x24, 0x14, 0x64, 0xee, 0x83, 0x23, 0x0f, 0xcd, 0x03, 0xaf,
0xe9, 0xc0, 0xbd, 0x07, 0xb0, 0xee, 0x0f, 0x7e, 0x19, 0xce, 0x53, 0xb6, 0x05, 0xb6, 0x3f, 0x50,
0x1b, 0x70, 0xe9, 0xf9, 0xf0, 0xc5, 0xd1, 0x55, 0x9a, 0x04, 0xa3, 0x54, 0x8c, 0xfd, 0x81, 0x84,
0x8f, 0x6d, 0x42, 0xcd, 0x1f, 0x90, 0xaf, 0x0e, 0xaf, 0xf9, 0x03, 0xb6, 0x0b, 0xce, 0x79, 0x10,
0x29, 0xe0, 0xb6, 0xb4, 0x73, 0xd2, 0x2c, 0x27, 0xa9, 0x77, 0x51, 0x30, 0x95, 0xe1, 0xf4, 0x15,
0xac, 0xbf, 0x0a, 0x45, 0x34, 0x96, 0x87, 0xb6, 0x78, 0x46, 0xb1, 0xa7, 0x3a, 0x75, 0xd2, 0xea,
0x77, 0xb5, 0xd5, 0x15, 0x87, 0xf2, 0xbc, 0x7a, 0x0f, 0xa1, 0xf1, 0x5a, 0x5c, 0x53, 0x2c, 0x2a,
0x52, 0xcb, 0x88, 0xf4, 0x5f, 0x16, 0xdc, 0xcf, 0x77, 0x9f, 0x05, 0x17, 0x91, 0x38, 0x0f, 0xa2,
0x85, 0x60, 0xbb, 0x2a, 0x6e, 0xab, 0xca, 0xff, 0xe3, 0x35, 0xc2, 0x82, 0x7d, 0x93, 0x63, 0x87,
0x6a, 0x5f, 0x68, 0xb5, 0xec, 0xc8, 0xe3, 0xb5, 0xac, 0x92, 0xb6, 0xa1, 0x79, 0x38, 0xf4, 0xc9,
0xb4, 0x6b, 0xf7, 0xac, 0xbe, 0x7d, 0xbc, 0xc6, 0x73, 0x0e, 0x7b, 0x00, 0x8d, 0x93, 0x45, 0x2a,
0xae, 0xfc, 0x01, 0x55, 0x90, 0x73, 0xbc, 0xc6, 0x15, 0x03, 0x77, 0xd2, 0xf2, 0xb5, 0xb8, 0x76,
0xeb, 0x3d, 0xab, 0xdf, 0xc2, 0x9d, 0x8a, 0xc3, 0xba, 0xe0, 0x1c, 0xc6, 0x71, 0xe4, 0xae, 0xf7,
0xac, 0x7e, 0x13, 0x4f, 0x43, 0xea, 0xb0, 0x01, 0x75, 0x32, 0xec, 0xfd, 0x01, 0xba, 0xc5, 0xe0,
0xb2, 0x74, 0x31, 0xb0, 0xd1, 0x9e, 0x95, 0xd9, 0x43, 0x82, 0x6d, 0x51, 0x0a, 0x6b, 0xd9, 0xf9,
0x98, 0xc4, 0xa7, 0xb0, 0x4e, 0x66, 0xe4, 0xa5, 0x68, 0x3f, 0x7e, 0x58, 0x01, 0xb8, 0x86, 0x8c,
0x67, 0xca, 0x87, 0x2d, 0x42, 0xfc, 0x57, 0x89, 0x3f, 0xf0, 0x7e, 0x5a, 0x06, 0x97, 0x72, 0x89,
0x89, 0x38, 0x0d, 0x26, 0x42, 0x9e, 0xcf, 0x69, 0x8d, 0xbc, 0xb3, 0xeb, 0x99, 0x20, 0x07, 0x5a,
0x9c, 0xd6, 0xde, 0x1f, 0x2d, 0xd8, 0x2c, 0xee, 0x47, 0x9f, 0x8c, 0xea, 0xb8, 0xc5, 0x27, 0xd2,
0xca, 0x8b, 0xe7, 0x79, 0xb9, 0x78, 0x76, 0x6e, 0xda, 0x57, 0xae, 0x9f, 0x9f, 0x81, 0xf3, 0x26,
0x08, 0x93, 0x95, 0x0a, 0xdf, 0x92, 0x10, 0xda, 0xe4, 0xae, 0x2d, 0x73, 0x51, 0x7f, 0x19, 0x2f,
0xa6, 0xa9, 0xc4, 0x90, 0x4b, 0xc2, 0x3b, 0x82, 0x16, 0xee, 0x97, 0x81, 0x7b, 0xd2, 0x58, 0x56,
0x56, 0x46, 0x3f, 0x41, 0x2e, 0x97, 0x07, 0x75, 0xa1, 0x4e, 0xca, 0x19, 0x12, 0x92, 0xf0, 0x8e,
// 1694 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x73, 0x1b, 0x4b,
0x11, 0xf7, 0x6a, 0x57, 0x96, 0xd4, 0x92, 0x1d, 0xbf, 0x89, 0xde, 0x63, 0x2b, 0x38, 0x7e, 0x62,
0xcb, 0xf0, 0x04, 0x45, 0x39, 0xe5, 0x90, 0xa4, 0x72, 0x01, 0x62, 0x47, 0x0e, 0xde, 0x0a, 0x36,
0x61, 0xe4, 0x72, 0x6e, 0x54, 0xad, 0xa5, 0xc1, 0xd9, 0x62, 0xa5, 0x15, 0xab, 0x55, 0x64, 0x5f,
0xa8, 0xe2, 0x33, 0xe4, 0xc2, 0x8d, 0x2b, 0x27, 0x3e, 0x04, 0x17, 0x38, 0x72, 0xa4, 0x8a, 0x0b,
0x15, 0xf8, 0x16, 0x5c, 0xa8, 0xee, 0xd9, 0xd9, 0x99, 0x5d, 0xad, 0x1d, 0x57, 0x8a, 0xdb, 0xf4,
0x9f, 0xe9, 0xe9, 0xfe, 0x75, 0x4f, 0x4f, 0xef, 0x42, 0x67, 0xb6, 0xb8, 0x88, 0xc2, 0xd1, 0xde,
0x2c, 0x89, 0xd3, 0x98, 0x35, 0xc3, 0x69, 0x2a, 0x92, 0x69, 0x10, 0x79, 0x7f, 0xb4, 0xc0, 0xe6,
0xf1, 0x92, 0xb9, 0xd0, 0x78, 0x19, 0x47, 0x8b, 0xc9, 0x74, 0xee, 0x5a, 0x3d, 0xbb, 0xef, 0x70,
0x45, 0x32, 0x06, 0xce, 0x6b, 0x71, 0x3d, 0x77, 0xed, 0x9e, 0xdd, 0x6f, 0x71, 0x5a, 0xb3, 0x5d,
0xa8, 0x1f, 0xa4, 0x69, 0x32, 0x77, 0x6b, 0x3d, 0xbb, 0xdf, 0x7e, 0xbc, 0xb9, 0xa7, 0xec, 0xed,
0x21, 0x9b, 0x4b, 0x21, 0xda, 0xe4, 0x71, 0x90, 0x84, 0xd3, 0x4b, 0xd7, 0xe9, 0x59, 0xfd, 0x0e,
0x57, 0x24, 0xeb, 0x42, 0xdd, 0x9f, 0x8e, 0xc5, 0x95, 0x5b, 0xef, 0x59, 0xfd, 0x16, 0x97, 0x04,
0x72, 0x5f, 0x85, 0x22, 0x1a, 0xbb, 0xeb, 0x92, 0x4b, 0x84, 0xb7, 0x07, 0x2d, 0x1e, 0x2f, 0x4f,
0x82, 0x34, 0x09, 0xaf, 0xd8, 0x77, 0xc0, 0xe1, 0xf1, 0x52, 0xfa, 0xd8, 0x7e, 0xbc, 0xa1, 0xcf,
0xe5, 0xf1, 0x92, 0x93, 0xc8, 0x3b, 0x81, 0xd6, 0x30, 0xbc, 0x9c, 0x8a, 0x31, 0x86, 0xf5, 0x35,
0xd8, 0x6f, 0x62, 0x54, 0xb7, 0x56, 0xd5, 0x51, 0x82, 0x0a, 0xa7, 0xe2, 0xd2, 0xad, 0x55, 0x2a,
0x9c, 0x8a, 0x4b, 0xef, 0x39, 0x6c, 0xf2, 0x78, 0xe9, 0x8f, 0xc5, 0x34, 0x0d, 0x7f, 0x1d, 0x8a,
0x84, 0x00, 0xc9, 0x7d, 0x70, 0xe4, 0xa1, 0x39, 0x48, 0x35, 0x0d, 0x92, 0xf7, 0x00, 0xd6, 0xfd,
0xc1, 0xcf, 0xc3, 0x79, 0xca, 0xb6, 0xc0, 0xf6, 0x07, 0x6a, 0x03, 0x2e, 0x3d, 0x1f, 0xbe, 0x38,
0xba, 0x4a, 0x93, 0x60, 0x94, 0x8a, 0xb1, 0x3f, 0x90, 0x50, 0xb3, 0x4d, 0xa8, 0xf9, 0x03, 0xf2,
0xd5, 0xe1, 0x35, 0x7f, 0xc0, 0x76, 0xc1, 0x39, 0x0f, 0x22, 0x05, 0xf2, 0x96, 0x76, 0x4e, 0x9a,
0xe5, 0x24, 0xf5, 0x2e, 0x0a, 0xa6, 0x32, 0x9c, 0xbe, 0x82, 0x75, 0x42, 0x4f, 0x1e, 0xda, 0xe2,
0x19, 0xc5, 0x9e, 0xea, 0x34, 0x4b, 0xab, 0xdf, 0xd6, 0x56, 0x57, 0x1c, 0xca, 0x6b, 0xc0, 0x7b,
0x08, 0x8d, 0xd7, 0xe2, 0x9a, 0x62, 0x51, 0x91, 0x5a, 0x46, 0xa4, 0xff, 0xb4, 0xe0, 0x7e, 0xbe,
0xfb, 0x2c, 0xb8, 0x88, 0xc4, 0x79, 0x10, 0x2d, 0x04, 0xdb, 0x55, 0x71, 0x5b, 0x55, 0xfe, 0x1f,
0xaf, 0x11, 0x16, 0xec, 0x9b, 0x1c, 0x3b, 0x54, 0xfb, 0x42, 0xab, 0x65, 0x47, 0x1e, 0xaf, 0x65,
0x55, 0xb7, 0x0d, 0xcd, 0xc3, 0xa1, 0x4f, 0xa6, 0x5d, 0xbb, 0x67, 0xf5, 0xed, 0xe3, 0x35, 0x9e,
0x73, 0xd8, 0x03, 0x68, 0x9c, 0x2c, 0x52, 0x71, 0xe5, 0x0f, 0xa8, 0xda, 0x9c, 0xe3, 0x35, 0xae,
0x18, 0xb8, 0x93, 0x96, 0xaf, 0xc5, 0xb5, 0x2c, 0x39, 0xdc, 0xa9, 0x38, 0xac, 0x0b, 0xce, 0x61,
0x1c, 0x47, 0x54, 0x76, 0x4d, 0x3c, 0x0d, 0xa9, 0xc3, 0x06, 0xd4, 0xc9, 0xb0, 0xf7, 0x3b, 0xe8,
0x16, 0x83, 0xcb, 0xd2, 0xc5, 0xc0, 0x46, 0x7b, 0x56, 0x66, 0x0f, 0x09, 0xb6, 0x45, 0x29, 0xac,
0x65, 0xe7, 0x63, 0x12, 0x9f, 0xc2, 0x3a, 0x99, 0x91, 0x17, 0xa8, 0xfd, 0xf8, 0x61, 0x05, 0xe0,
0x1a, 0x32, 0x9e, 0x29, 0x1f, 0xb6, 0x08, 0xf1, 0x5f, 0x24, 0xfe, 0xc0, 0xfb, 0x71, 0x19, 0x5c,
0xca, 0x25, 0x26, 0xe2, 0x34, 0x98, 0x08, 0x79, 0x3e, 0xa7, 0x35, 0xf2, 0xce, 0xae, 0x67, 0x82,
0x1c, 0x68, 0x71, 0x5a, 0x7b, 0xbf, 0xb7, 0x60, 0xb3, 0xb8, 0x1f, 0x7d, 0x32, 0xaa, 0xe3, 0x16,
0x9f, 0x48, 0x2b, 0x2f, 0x9e, 0xe7, 0xe5, 0xe2, 0xd9, 0xb9, 0x69, 0x5f, 0xb9, 0x7e, 0x7e, 0x02,
0xce, 0x9b, 0x20, 0x4c, 0x56, 0x2a, 0x7c, 0x4b, 0x42, 0x68, 0x93, 0xbb, 0xb6, 0xcc, 0x45, 0xfd,
0x65, 0xbc, 0x98, 0xa6, 0x12, 0x43, 0x2e, 0x09, 0xef, 0x08, 0x5a, 0xb8, 0x5f, 0x06, 0xee, 0x49,
0x63, 0x59, 0x59, 0x19, 0xbd, 0x07, 0xb9, 0x5c, 0x1e, 0x94, 0xb7, 0x92, 0x9a, 0xd9, 0x4a, 0x8e,
0x01, 0x50, 0x3a, 0x97, 0x76, 0x76, 0xa1, 0x4e, 0x54, 0x06, 0x42, 0xd9, 0x90, 0x14, 0xde, 0x60,
0xe9, 0x21, 0xd4, 0xfd, 0x69, 0xfa, 0xec, 0x09, 0x8a, 0x65, 0x41, 0xa2, 0x37, 0x36, 0xcf, 0x4a,
0x66, 0x01, 0x4d, 0x09, 0x5d, 0xbc, 0xd4, 0x06, 0x2c, 0xc3, 0x00, 0x72, 0xb1, 0xad, 0x0c, 0x54,
0x9c, 0x44, 0xe0, 0xb5, 0xe5, 0xf1, 0x52, 0x43, 0x92, 0x51, 0xec, 0xfb, 0xea, 0x14, 0x87, 0x62,
0xbe, 0x67, 0x5c, 0x25, 0xf4, 0x42, 0x1d, 0xfb, 0x1b, 0x80, 0x5f, 0x24, 0xf1, 0x62, 0x46, 0xa0,
0xb1, 0x3e, 0xd4, 0x89, 0xca, 0xe2, 0x63, 0x7a, 0x93, 0xf2, 0x8d, 0x4b, 0x85, 0x6a, 0xd0, 0x31,
0x39, 0xc3, 0xc5, 0x44, 0xde, 0x34, 0x8e, 0x4b, 0x2c, 0xa5, 0xe6, 0x79, 0x10, 0xe5, 0xe2, 0xf3,
0x20, 0xca, 0xe2, 0xc6, 0x65, 0xd1, 0x8c, 0xad, 0xcc, 0x3c, 0x80, 0xe6, 0xab, 0x28, 0x0e, 0x52,
0x54, 0x46, 0x5b, 0x16, 0xcf, 0x69, 0xb6, 0x0f, 0x30, 0x10, 0xa3, 0x70, 0x12, 0x44, 0x28, 0x75,
0xca, 0x0d, 0x20, 0x93, 0x71, 0x43, 0xc9, 0x7b, 0x0a, 0x8d, 0x8c, 0xaa, 0xc6, 0x1e, 0xb9, 0xc3,
0x51, 0x10, 0x09, 0xe5, 0x05, 0x11, 0xde, 0x5b, 0xd8, 0x90, 0xc5, 0x88, 0xcf, 0xcd, 0x50, 0xa4,
0x77, 0x28, 0xc5, 0x3b, 0x3d, 0x5c, 0xde, 0x9f, 0x2d, 0x70, 0x70, 0xa5, 0x0c, 0x58, 0xda, 0x80,
0x79, 0x1b, 0x1d, 0x79, 0x1b, 0x59, 0x0f, 0xda, 0xc3, 0x14, 0xdf, 0x35, 0xdd, 0xc6, 0x5a, 0xdc,
0x64, 0x21, 0x5e, 0xfe, 0x34, 0xd5, 0xe9, 0xb6, 0x79, 0x4e, 0xb3, 0x6d, 0x68, 0x61, 0x6f, 0x92,
0x42, 0x6c, 0x64, 0x4d, 0xae, 0x19, 0x6c, 0x07, 0x40, 0x21, 0xbb, 0x10, 0xd4, 0xcd, 0x2c, 0x6e,
0x70, 0xbc, 0x47, 0xd0, 0x40, 0x4f, 0x4f, 0x82, 0x99, 0x8e, 0xcd, 0xba, 0x2d, 0xb6, 0xff, 0x59,
0xd0, 0xf9, 0xf5, 0x42, 0x24, 0xd7, 0x5c, 0xfc, 0x7e, 0x21, 0xe6, 0x29, 0x62, 0x4b, 0xb4, 0xaa,
0x65, 0x22, 0xb0, 0x6a, 0x87, 0xef, 0x82, 0x64, 0x2c, 0x91, 0x72, 0x78, 0x46, 0x61, 0xac, 0x1a,
0xf3, 0x39, 0xc5, 0xda, 0xe4, 0x26, 0x8b, 0xea, 0x5d, 0x4c, 0xe2, 0x54, 0x05, 0x93, 0x51, 0xac,
0x0f, 0xf7, 0x8e, 0xae, 0x46, 0xd1, 0x62, 0x2c, 0x78, 0xbc, 0x94, 0xbb, 0xa9, 0x39, 0xf3, 0x32,
0x9b, 0xfd, 0x00, 0x9b, 0x1b, 0xb1, 0x54, 0x6b, 0x6a, 0x90, 0x62, 0x89, 0xcb, 0xf6, 0xa1, 0x73,
0x34, 0xb9, 0x10, 0xe3, 0xb1, 0x18, 0x0f, 0x82, 0x34, 0x70, 0x9b, 0x55, 0x03, 0x44, 0x41, 0xc5,
0xfb, 0x60, 0xc1, 0x46, 0x16, 0xfd, 0x7c, 0x16, 0x4f, 0xe7, 0x02, 0x53, 0x7c, 0x94, 0x24, 0x2a,
0xc5, 0x47, 0x49, 0xc2, 0x1e, 0x41, 0x83, 0x8b, 0xf9, 0x22, 0x4a, 0x55, 0x95, 0x7c, 0xa9, 0x2d,
0xaa, 0xbd, 0x8b, 0x28, 0xe5, 0x4a, 0x8b, 0xfd, 0x1c, 0x36, 0x0b, 0x75, 0xa8, 0x9e, 0x85, 0xef,
0xe8, 0x7d, 0x05, 0x39, 0x2f, 0xa9, 0x7b, 0x7f, 0xa9, 0x43, 0xdb, 0xb0, 0x9c, 0x17, 0x19, 0xe2,
0xb3, 0x91, 0x15, 0xd9, 0xd7, 0x34, 0xa7, 0xdd, 0x30, 0xf5, 0x60, 0x4f, 0xea, 0x80, 0x75, 0x9a,
0x95, 0xa5, 0x75, 0xaa, 0x1b, 0xa1, 0x7d, 0x5b, 0x23, 0xc4, 0xa9, 0xef, 0x5d, 0x30, 0xbd, 0x14,
0x63, 0x2a, 0xcb, 0x26, 0x57, 0x24, 0xdb, 0xd3, 0x5d, 0x81, 0xf2, 0x58, 0xe8, 0x35, 0x4a, 0xc2,
0x75, 0xe7, 0x90, 0x5d, 0x0e, 0x27, 0x83, 0x86, 0xac, 0x17, 0x49, 0xb1, 0x67, 0xd0, 0xd6, 0xed,
0x6b, 0x9e, 0xa5, 0xa8, 0xab, 0x4d, 0x69, 0x21, 0x37, 0x15, 0xd9, 0x8b, 0xf2, 0x88, 0xe6, 0xb6,
0xc8, 0x0b, 0xb7, 0x10, 0xb9, 0x21, 0xe7, 0xe5, 0x91, 0x6e, 0xdf, 0x98, 0x19, 0x5d, 0xa0, 0xcd,
0xf7, 0xf5, 0xe6, 0x5c, 0xc4, 0x8d, 0xc9, 0xf2, 0x89, 0xf9, 0x96, 0xb8, 0x6d, 0xda, 0xd3, 0x2d,
0x22, 0x27, 0x65, 0xdc, 0x7c, 0x73, 0xf6, 0x8d, 0x87, 0xcc, 0xed, 0x94, 0x0f, 0xca, 0x45, 0xdc,
0x78, 0xee, 0xfc, 0x8a, 0xf9, 0xce, 0xdd, 0xa0, 0xad, 0xd5, 0xc3, 0x9b, 0x54, 0xe1, 0x15, 0x53,
0xe1, 0x8b, 0xf2, 0x24, 0xe0, 0x6e, 0x96, 0x81, 0x2a, 0xca, 0x79, 0x79, 0x72, 0xd8, 0x37, 0x86,
0x71, 0xf7, 0x5e, 0xd9, 0xff, 0x5c, 0xc4, 0xb5, 0x96, 0xf7, 0xb7, 0x1a, 0x6c, 0xf8, 0x93, 0x59,
0x9c, 0xa4, 0x46, 0x17, 0xf1, 0xa7, 0x63, 0x71, 0xa5, 0xba, 0x08, 0x11, 0xd5, 0x0f, 0x2d, 0x75,
0x73, 0xec, 0x26, 0xd4, 0x3d, 0x1c, 0x2e, 0x09, 0xa3, 0x82, 0x9c, 0x42, 0x05, 0x6d, 0x43, 0x4b,
0x5e, 0x17, 0x14, 0xd5, 0x49, 0xa4, 0x19, 0xf2, 0x1b, 0x63, 0x49, 0xb3, 0x66, 0x83, 0xa6, 0x57,
0x45, 0x62, 0xe7, 0x94, 0x6a, 0x24, 0x6c, 0x92, 0xd0, 0xe0, 0xa0, 0xfc, 0x2c, 0x9c, 0x88, 0x79,
0x1a, 0x4c, 0x66, 0xd8, 0x8a, 0xec, 0xbe, 0xcd, 0x0d, 0x0e, 0x76, 0x21, 0x0a, 0xe2, 0x65, 0x22,
0x82, 0x54, 0x8c, 0x0f, 0x52, 0xaa, 0x40, 0x9b, 0x97, 0xb8, 0xa8, 0x47, 0x61, 0x69, 0x3d, 0x90,
0x7a, 0x45, 0x2e, 0xbd, 0xa4, 0x91, 0x08, 0x12, 0xaa, 0xab, 0x26, 0x97, 0x84, 0xf7, 0xcf, 0x1a,
0x30, 0x89, 0xa4, 0x9c, 0x15, 0xbf, 0x35, 0x38, 0x6f, 0x87, 0xad, 0x08, 0x4e, 0x63, 0x05, 0x9c,
0xaf, 0xf2, 0x09, 0x57, 0x02, 0x93, 0x51, 0xd8, 0xfe, 0xf5, 0xe3, 0x23, 0x51, 0xb5, 0xb8, 0xc9,
0x62, 0x1e, 0x74, 0x8c, 0x97, 0x0f, 0xaf, 0x2d, 0xda, 0x2e, 0xf0, 0x2a, 0xa0, 0x85, 0x3b, 0x42,
0xdb, 0xbe, 0x1d, 0xda, 0x8e, 0x09, 0xed, 0x07, 0x0b, 0x3a, 0x07, 0x69, 0x3c, 0x09, 0x47, 0x5c,
0x8c, 0xe2, 0x64, 0x7c, 0x33, 0xa8, 0x12, 0xbe, 0x9a, 0x09, 0xdf, 0x1e, 0xd8, 0xfe, 0xfb, 0x24,
0xeb, 0x9e, 0xdb, 0xc6, 0x6c, 0xb6, 0x92, 0x2b, 0x8e, 0x8a, 0xec, 0x1b, 0xa8, 0xf9, 0x09, 0x55,
0x6e, 0xa1, 0xef, 0x17, 0x2e, 0x09, 0xaf, 0xf9, 0x89, 0xf7, 0x63, 0xe8, 0x4a, 0xa7, 0x94, 0x28,
0x7b, 0x87, 0xba, 0x50, 0x3f, 0x4a, 0x92, 0x58, 0xbd, 0x44, 0x92, 0xf0, 0xae, 0xa0, 0x7b, 0x96,
0x04, 0xd3, 0x79, 0x14, 0xa4, 0x02, 0x13, 0xf3, 0x39, 0xf5, 0x51, 0xf5, 0x01, 0xdf, 0x83, 0xf6,
0x69, 0x9c, 0xbe, 0x4d, 0xc2, 0x94, 0x5a, 0x86, 0x6c, 0xfe, 0x26, 0xcb, 0xfb, 0x21, 0x7c, 0x59,
0x3a, 0x59, 0x3f, 0x98, 0x58, 0x52, 0xb6, 0xfe, 0xf0, 0x1d, 0xc2, 0xfd, 0x5c, 0xd5, 0x1f, 0x7c,
0x96, 0x8f, 0xab, 0x46, 0x7f, 0x64, 0x44, 0x4e, 0x46, 0xb3, 0xe3, 0x2b, 0xa2, 0xf1, 0x0e, 0xc1,
0xcd, 0xd0, 0x94, 0xff, 0x17, 0x32, 0x0f, 0xce, 0x43, 0xb1, 0xbc, 0xe9, 0x93, 0x8a, 0x06, 0x86,
0x1a, 0xfd, 0x95, 0xa0, 0xb5, 0xf7, 0x5f, 0x0b, 0xba, 0x55, 0x46, 0x74, 0x71, 0x59, 0x46, 0x71,
0xb1, 0xe7, 0x50, 0x7f, 0x1f, 0x8a, 0xa5, 0x1a, 0x11, 0xbc, 0x95, 0x94, 0xaf, 0x78, 0xc2, 0xe5,
0x06, 0xbc, 0x5a, 0x07, 0xa3, 0x34, 0x8c, 0xa7, 0xea, 0x7b, 0x40, 0x52, 0x78, 0xce, 0x61, 0x14,
0x8f, 0x7e, 0x27, 0xbf, 0x74, 0xb9, 0x24, 0x2a, 0xae, 0x4a, 0xfd, 0x8e, 0x57, 0x65, 0xbd, 0xea,
0xaa, 0x78, 0x7f, 0xb5, 0x14, 0x56, 0xc6, 0xcc, 0xf6, 0xc9, 0x8c, 0xe9, 0x0b, 0x62, 0xab, 0x0b,
0xe2, 0xca, 0xc1, 0x53, 0xcf, 0xd7, 0x8a, 0xc4, 0x61, 0x17, 0x97, 0xf4, 0x9b, 0xc3, 0xa1, 0x2c,
0xe5, 0xf4, 0x27, 0xba, 0xd2, 0x6a, 0xb0, 0xeb, 0x55, 0xc1, 0x1e, 0x6e, 0xfd, 0xfd, 0xe3, 0x8e,
0xf5, 0x8f, 0x8f, 0x3b, 0xd6, 0xbf, 0x3f, 0xee, 0x58, 0x7f, 0xfa, 0xcf, 0xce, 0xda, 0xc5, 0x3a,
0xfd, 0xd6, 0xfa, 0xc9, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0xf4, 0x59, 0xf3, 0x5b, 0xe6, 0x12,
0x00, 0x00,
0xe9, 0x21, 0x36, 0xb0, 0xf4, 0xd9, 0x13, 0x14, 0xcb, 0x82, 0x44, 0x6f, 0x6c, 0x9e, 0x95, 0xcc,
0x02, 0x9a, 0x12, 0xba, 0x78, 0xa9, 0x0d, 0x58, 0x86, 0x01, 0xe4, 0x62, 0x5b, 0x19, 0xa8, 0x38,
0x89, 0xc0, 0x6b, 0xcb, 0xe3, 0xa5, 0x86, 0x24, 0xa3, 0xd8, 0x77, 0xd5, 0x29, 0x0e, 0xc5, 0x7c,
0xcf, 0xb8, 0x4a, 0xe8, 0x85, 0x3a, 0xf6, 0x57, 0x00, 0x3f, 0x4b, 0xe2, 0xc5, 0x8c, 0x40, 0x63,
0x7d, 0xa8, 0x13, 0x95, 0xc5, 0xc7, 0xf4, 0x26, 0xe5, 0x1b, 0x97, 0x0a, 0xd5, 0xa0, 0x63, 0x72,
0x86, 0x8b, 0x89, 0xbc, 0x69, 0x1c, 0x97, 0x58, 0x4a, 0xcd, 0xf3, 0x20, 0xca, 0xc5, 0xe7, 0x41,
0x94, 0xc5, 0x8d, 0xcb, 0xa2, 0x19, 0x5b, 0x99, 0x79, 0x00, 0xcd, 0x57, 0x51, 0x1c, 0xa4, 0xa8,
0x8c, 0xb6, 0x2c, 0x9e, 0xd3, 0x6c, 0x1f, 0x60, 0x20, 0x46, 0xe1, 0x24, 0x88, 0x50, 0xea, 0x94,
0x1b, 0x40, 0x26, 0xe3, 0x86, 0x92, 0xf7, 0x14, 0x1a, 0x19, 0x55, 0x8d, 0x3d, 0x72, 0x87, 0xa3,
0x20, 0x12, 0xca, 0x0b, 0x22, 0xbc, 0xb7, 0xb0, 0x21, 0x8b, 0x11, 0x9f, 0xa6, 0xa1, 0x48, 0xef,
0x50, 0x8a, 0x77, 0x7a, 0xe4, 0xbc, 0x3f, 0x59, 0xe0, 0xe0, 0x4a, 0x19, 0xb0, 0xb4, 0x01, 0xf3,
0x36, 0x3a, 0xf2, 0x36, 0xb2, 0x1e, 0xb4, 0x87, 0x29, 0xbe, 0x81, 0xba, 0x8d, 0xb5, 0xb8, 0xc9,
0x42, 0xbc, 0xfc, 0x69, 0xaa, 0xd3, 0x6d, 0xf3, 0x9c, 0x66, 0xdb, 0xd0, 0xc2, 0xde, 0x24, 0x85,
0xd8, 0xc8, 0x9a, 0x5c, 0x33, 0xd8, 0x0e, 0x80, 0x42, 0x76, 0x21, 0xa8, 0x9b, 0x59, 0xdc, 0xe0,
0x78, 0x8f, 0xa0, 0x81, 0x9e, 0x9e, 0x04, 0x33, 0x1d, 0x9b, 0x75, 0x5b, 0x6c, 0xff, 0xb5, 0xa0,
0xf3, 0xcb, 0x85, 0x48, 0xae, 0xb9, 0xf8, 0xed, 0x42, 0xcc, 0x53, 0xc4, 0x96, 0x68, 0x55, 0xcb,
0x44, 0x60, 0xd5, 0x0e, 0xdf, 0x05, 0xc9, 0x58, 0x22, 0xe5, 0xf0, 0x8c, 0xc2, 0x58, 0x35, 0xe6,
0x73, 0x8a, 0xb5, 0xc9, 0x4d, 0x16, 0xd5, 0xbb, 0x98, 0xc4, 0xa9, 0x0a, 0x26, 0xa3, 0x58, 0x1f,
0xee, 0x1d, 0x5d, 0x8d, 0xa2, 0xc5, 0x58, 0xf0, 0x78, 0x29, 0x77, 0x53, 0x73, 0xe6, 0x65, 0x36,
0xfb, 0x1e, 0x36, 0x37, 0x62, 0xa9, 0xd6, 0xd4, 0x20, 0xc5, 0x12, 0x97, 0xed, 0x43, 0xe7, 0x68,
0x72, 0x21, 0xc6, 0x63, 0x31, 0x1e, 0x04, 0x69, 0xe0, 0x36, 0xab, 0x06, 0x88, 0x82, 0x8a, 0xf7,
0xc1, 0x82, 0x8d, 0x2c, 0xfa, 0xf9, 0x2c, 0x9e, 0xce, 0x05, 0xa6, 0xf8, 0x28, 0x49, 0x54, 0x8a,
0x8f, 0x92, 0x84, 0x3d, 0x82, 0x06, 0x17, 0xf3, 0x45, 0x94, 0xaa, 0x2a, 0xf9, 0x52, 0x5b, 0x54,
0x7b, 0x17, 0x51, 0xca, 0x95, 0x16, 0xfb, 0x29, 0x6c, 0x16, 0xea, 0x50, 0x3d, 0x0b, 0xdf, 0xd2,
0xfb, 0x0a, 0x72, 0x5e, 0x52, 0xf7, 0xfe, 0x5c, 0x87, 0xb6, 0x61, 0x39, 0x2f, 0x32, 0xc4, 0x67,
0x23, 0x2b, 0xb2, 0xaf, 0x69, 0xa6, 0xbb, 0x61, 0xea, 0xc1, 0x9e, 0xd4, 0x01, 0xeb, 0x34, 0x2b,
0x4b, 0xeb, 0x54, 0x37, 0x42, 0xfb, 0xb6, 0x46, 0x88, 0x13, 0xe2, 0xbb, 0x60, 0x7a, 0x29, 0xc6,
0x54, 0x96, 0x4d, 0xae, 0x48, 0xb6, 0xa7, 0xbb, 0x02, 0xe5, 0xb1, 0xd0, 0x6b, 0x94, 0x84, 0xeb,
0xce, 0x21, 0xbb, 0x1c, 0x4e, 0x06, 0x0d, 0x59, 0x2f, 0x92, 0x62, 0xcf, 0xa0, 0xad, 0xdb, 0xd7,
0x3c, 0x4b, 0x51, 0x57, 0x9b, 0xd2, 0x42, 0x6e, 0x2a, 0xb2, 0x17, 0xe5, 0x11, 0xcd, 0x6d, 0x91,
0x17, 0x6e, 0x21, 0x72, 0x43, 0xce, 0xcb, 0x23, 0xdd, 0xbe, 0x31, 0x33, 0xba, 0x40, 0x9b, 0xef,
0xeb, 0xcd, 0xb9, 0x88, 0x1b, 0x93, 0xe5, 0x13, 0xf3, 0x2d, 0x71, 0xdb, 0xb4, 0xa7, 0x5b, 0x44,
0x4e, 0xca, 0xb8, 0xf9, 0xe6, 0xec, 0x1b, 0x0f, 0x99, 0xdb, 0x29, 0x1f, 0x94, 0x8b, 0xb8, 0xf1,
0xdc, 0xf9, 0x15, 0xf3, 0x9d, 0xbb, 0x41, 0x5b, 0xab, 0x87, 0x37, 0xa9, 0xc2, 0x2b, 0xa6, 0xc2,
0x17, 0xe5, 0x49, 0xc0, 0xdd, 0x2c, 0x03, 0x55, 0x94, 0xf3, 0xf2, 0xe4, 0xb0, 0x6f, 0x0c, 0xe3,
0xee, 0xbd, 0xb2, 0xff, 0xb9, 0x88, 0x6b, 0x2d, 0xef, 0xaf, 0x35, 0xd8, 0xf0, 0x27, 0xb3, 0x38,
0x49, 0x8d, 0x2e, 0x22, 0xa7, 0x7f, 0xab, 0x72, 0xfa, 0xaf, 0x95, 0xde, 0x49, 0xea, 0x26, 0xd4,
0x3d, 0x1c, 0x2e, 0x09, 0xa3, 0x82, 0x9c, 0x42, 0x05, 0x6d, 0x43, 0x4b, 0x5e, 0x17, 0x14, 0xd5,
0x49, 0xa4, 0x19, 0xf2, 0x7b, 0x64, 0x49, 0xb3, 0x66, 0x83, 0xa6, 0x57, 0x45, 0x62, 0xe7, 0x94,
0x6a, 0x24, 0x6c, 0x92, 0xd0, 0xe0, 0xa0, 0xfc, 0x2c, 0x9c, 0x88, 0x79, 0x1a, 0x4c, 0x66, 0xd8,
0x8a, 0xec, 0xbe, 0xcd, 0x0d, 0x0e, 0x76, 0x21, 0x0a, 0xe2, 0x65, 0x22, 0x82, 0x54, 0x8c, 0x0f,
0x52, 0xaa, 0x40, 0x9b, 0x97, 0xb8, 0xa8, 0x47, 0x61, 0x69, 0x3d, 0x90, 0x7a, 0x45, 0x2e, 0xbd,
0xa4, 0x91, 0x08, 0x12, 0xaa, 0xab, 0x26, 0x97, 0x84, 0xf7, 0x8f, 0x1a, 0x30, 0x89, 0xa4, 0x9c,
0x15, 0xff, 0x6f, 0x70, 0xde, 0x0e, 0x5b, 0x11, 0x9c, 0xc6, 0x0a, 0x38, 0x5f, 0xe5, 0x13, 0xae,
0x04, 0x26, 0xa3, 0xb0, 0xfd, 0xeb, 0xc7, 0x47, 0xa2, 0x6a, 0x71, 0x93, 0xc5, 0x3c, 0xe8, 0x18,
0x2f, 0x1f, 0x5e, 0x5b, 0xb4, 0x5d, 0xe0, 0x55, 0x40, 0x0b, 0x77, 0x84, 0xb6, 0x7d, 0x3b, 0xb4,
0x1d, 0x13, 0xda, 0x0f, 0x16, 0x74, 0x0e, 0xd2, 0x78, 0x12, 0x8e, 0xb8, 0x18, 0xc5, 0xc9, 0xf8,
0x66, 0x50, 0x25, 0x7c, 0x35, 0x13, 0xbe, 0x3d, 0xb0, 0xfd, 0xf7, 0x49, 0xd6, 0x3d, 0xb7, 0x8d,
0xd9, 0x6c, 0x25, 0x57, 0x1c, 0x15, 0xd9, 0x37, 0x50, 0xf3, 0x13, 0xaa, 0xdc, 0x42, 0xdf, 0x2f,
0x5c, 0x12, 0x5e, 0xf3, 0x13, 0xef, 0x87, 0xd0, 0x95, 0x4e, 0x29, 0x51, 0xf6, 0x0e, 0x75, 0xa1,
0x7e, 0x94, 0x24, 0xb1, 0x7a, 0x89, 0x24, 0xe1, 0x5d, 0x41, 0xf7, 0x2c, 0x09, 0xa6, 0xf3, 0x28,
0x48, 0x05, 0x26, 0xe6, 0x73, 0xea, 0xa3, 0xea, 0x63, 0xbf, 0x07, 0xed, 0xd3, 0x38, 0x7d, 0x9b,
0x84, 0x29, 0xb5, 0x0c, 0xd9, 0xfc, 0x4d, 0x96, 0xf7, 0x7d, 0xf8, 0xb2, 0x74, 0xb2, 0x7e, 0x30,
0xb1, 0xa4, 0x6c, 0xfd, 0xe1, 0x3b, 0x84, 0xfb, 0xb9, 0xaa, 0x3f, 0xf8, 0x2c, 0x1f, 0x57, 0x8d,
0xfe, 0xc0, 0x88, 0x9c, 0x8c, 0x66, 0xc7, 0x57, 0x44, 0xe3, 0x1d, 0x82, 0x9b, 0xa1, 0x29, 0xff,
0x45, 0x64, 0x1e, 0x9c, 0x87, 0x62, 0x79, 0xd3, 0x27, 0x15, 0x0d, 0x0c, 0x35, 0xfa, 0x83, 0x41,
0x6b, 0xef, 0x3f, 0x16, 0x74, 0xab, 0x8c, 0xe8, 0xe2, 0xb2, 0x8c, 0xe2, 0x62, 0xcf, 0xa1, 0xfe,
0x3e, 0x14, 0x4b, 0x35, 0x22, 0x78, 0x2b, 0x29, 0x5f, 0xf1, 0x84, 0xcb, 0x0d, 0x78, 0xb5, 0x0e,
0x46, 0x69, 0x18, 0x4f, 0xd5, 0xf7, 0x80, 0xa4, 0xf0, 0x9c, 0xc3, 0x28, 0x1e, 0xfd, 0x46, 0x7e,
0xe9, 0x72, 0x49, 0x54, 0x5c, 0x95, 0xfa, 0x1d, 0xaf, 0xca, 0x7a, 0xd5, 0x55, 0xf1, 0xfe, 0x62,
0x29, 0xac, 0x8c, 0x99, 0xed, 0x93, 0x19, 0xd3, 0x17, 0xc4, 0x56, 0x17, 0xc4, 0x95, 0x83, 0xa7,
0x9e, 0xaf, 0x15, 0x89, 0xc3, 0x2e, 0x2e, 0xe9, 0x37, 0x87, 0x43, 0x59, 0xca, 0xe9, 0x4f, 0x74,
0xa5, 0xd5, 0x60, 0xd7, 0xab, 0x82, 0x3d, 0xdc, 0xfa, 0xdb, 0xc7, 0x1d, 0xeb, 0xef, 0x1f, 0x77,
0xac, 0x7f, 0x7d, 0xdc, 0xb1, 0xfe, 0xf0, 0xef, 0x9d, 0xb5, 0x8b, 0x75, 0xfa, 0x07, 0xf6, 0xa3,
0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x2d, 0xb9, 0x97, 0xfb, 0x13, 0x13, 0x00, 0x00,
}
func (m *Row) Marshal() (dAtA []byte, err error) {
@ -2800,6 +2816,20 @@ func (m *Row) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if len(m.Field) > 0 {
i -= len(m.Field)
copy(dAtA[i:], m.Field)
i = encodeVarintPublic(dAtA, i, uint64(len(m.Field)))
i--
dAtA[i] = 0x32
}
if len(m.Index) > 0 {
i -= len(m.Index)
copy(dAtA[i:], m.Index)
i = encodeVarintPublic(dAtA, i, uint64(len(m.Index)))
i--
dAtA[i] = 0x2a
}
if len(m.Roaring) > 0 {
i -= len(m.Roaring)
copy(dAtA[i:], m.Roaring)
@ -5183,6 +5213,14 @@ func (m *Row) Size() (n int) {
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
l = len(m.Field)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -6457,6 +6495,70 @@ func (m *Row) Unmarshal(dAtA []byte) error {
m.Roaring = []byte{}
}
iNdEx = postIndex
case 5:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthPublic
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 6:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthPublic
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Field = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])

2
internal/public.proto
View file

@ -7,6 +7,8 @@ message Row {
repeated string Keys = 3;
repeated Attr Attrs = 2;
bytes Roaring = 4;
string Index = 5;
string Field = 6;
}
message RowMatrix {

13
row.go
View file

@ -33,6 +33,15 @@ type Row struct {
// Attributes associated with the row.
Attrs map[string]interface{}
// Index tells what index this row is from - needed for key translation.
Index string
// Field tells what field this row is from if it's a "vertical"
// row. It may be the result of a Distinct query or Rows
// query. Knowing the index and field, we can figure out how to
// interpret the row data.
Field string
}
// NewRow returns a new instance of Row.
@ -61,6 +70,8 @@ func (r *Row) Clone() (clone *Row) {
clone = &Row{
Keys: keyClone,
Attrs: attrClone,
Index: r.Index,
Field: r.Field,
}
for _, seg := range r.segments {
@ -322,7 +333,7 @@ func (r *Row) Union(others ...*Row) *Row {
output = append(output, *toProcess[0].Union(toProcess[1:]...))
}
}
return &Row{segments: output}
return &Row{Index: r.Index, Field: r.Field, segments: output}
}
// Difference returns the diff of r and other.

8
rrtx.go
View file

@ -30,6 +30,7 @@ import (
rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg"
"github.com/pilosa/pilosa/v2/roaring"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
//txkey "github.com/pilosa/pilosa/v2/txkey"
"github.com/pkg/errors"
)
@ -372,13 +373,13 @@ func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*frag
v := f.view(view)
if v == nil {
return nil, errors.Errorf("view not found: %q", view)
return nil, errors.Wrapf(ViewNotFound, "getting %s", view)
}
frag := v.Fragment(shard)
if frag == nil {
return nil, fmt.Errorf("fragment not found: %q / %q / %d", field, view, shard)
return nil, errors.Wrapf(FragmentNotFound, "field:%q, view:%q, shard:%d", field, view, shard)
}
// Note: we cannot cache frag into tx.fragment.
@ -388,6 +389,9 @@ func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*frag
return frag, nil
}
const ViewNotFound = Error("view not found")
const FragmentNotFound = Error("fragment not found")
func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
frag, err := tx.getFragment(index, field, view, shard)
if err != nil {

4
server.go
View file

@ -499,6 +499,10 @@ func (s *Server) InternalClient() InternalClient {
return s.defaultClient
}
func (s *Server) GRPCURI() URI {
return s.grpcURI
}
// UpAndDown brings the server up minimally and shuts it down
// again; basically, it exists for testing holder open and close.
func (s *Server) UpAndDown() error {

107
test/cluster.go
View file

@ -18,6 +18,7 @@ import (
"context"
"fmt"
"io/ioutil"
"math"
"path"
"strconv"
"strings"
@ -25,6 +26,8 @@ import (
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/api/client"
"github.com/pilosa/pilosa/v2/proto"
"github.com/pilosa/pilosa/v2/server"
"github.com/pkg/errors"
)
@ -52,6 +55,39 @@ func (c *Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse
return c.Nodes[0].QueryAPI(t, &pilosa.QueryRequest{Index: index, Query: query})
}
// QueryHTTP executes a PQL query through the HTTP endpoint. It fails
// the test for explicit errors, but returns an error which has the
// response body if the HTTP call returns a non-OK status.
func (c *Cluster) QueryHTTP(t testing.TB, index, query string) (string, error) {
t.Helper()
if len(c.Nodes) == 0 {
t.Fatal("must have at least one node in cluster to query")
}
return c.Nodes[0].Query(t, index, "", query)
}
// QueryGRPC executes a PQL query through the GRPC endpoint. It fails the
// test if there is an error.
func (c *Cluster) QueryGRPC(t testing.TB, index, query string) *proto.TableResponse {
t.Helper()
if len(c.Nodes) == 0 {
t.Fatal("must have at least one node in cluster to query")
}
grpcClient, err := client.NewGRPCClient([]string{fmt.Sprintf("%s:%d", c.Nodes[0].Server.GRPCURI().Host, c.Nodes[0].Server.GRPCURI().Port)}, nil)
if err != nil {
t.Fatalf("getting GRPC client: %v", err)
}
tableResp, err := grpcClient.QueryUnary(context.Background(), index, query)
if err != nil {
t.Fatalf("querying unary: %v", err)
}
return tableResp
}
func (c *Cluster) GetNode(n int) *Command {
return c.Nodes[n]
}
@ -108,6 +144,77 @@ func (c *Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uin
}
}
// ImportKeyKey imports data into an index where both the index and
// the field are using string keys.
func (c *Cluster) ImportKeyKey(t testing.TB, index, field string, valAndRecKeys [][2]string) {
t.Helper()
importRequest := &pilosa.ImportRequest{
Index: index,
Field: field,
RowKeys: make([]string, len(valAndRecKeys)),
ColumnKeys: make([]string, len(valAndRecKeys)),
}
for i, vk := range valAndRecKeys {
importRequest.RowKeys[i] = vk[0]
importRequest.ColumnKeys[i] = vk[1]
}
err := c.Nodes[0].API.Import(context.Background(), nil, importRequest)
if err != nil {
t.Fatalf("importing keykey data: %v", err)
}
}
// IntKey is a string key and a signed integer value.
type IntKey struct {
Val int64
Key string
}
// ImportIntKey imports int data into an index which uses string keys.
func (c *Cluster) ImportIntKey(t testing.TB, index, field string, pairs []IntKey) {
t.Helper()
importRequest := &pilosa.ImportValueRequest{
Index: index,
Field: field,
Shard: math.MaxUint64,
ColumnKeys: make([]string, len(pairs)),
Values: make([]int64, len(pairs)),
}
for i, pair := range pairs {
importRequest.Values[i] = pair.Val
importRequest.ColumnKeys[i] = pair.Key
}
if err := c.Nodes[0].API.ImportValue(context.Background(), nil, importRequest); err != nil {
t.Fatalf("importing IntKey data: %v", err)
}
}
// KeyID represents a key and an ID for importing data into an index
// and field where one uses string keys and the other does not.
type KeyID struct {
Key string
ID uint64
}
//ImportIDKey imports data into an unkeyed set field in a keyed index.
func (c *Cluster) ImportIDKey(t testing.TB, index, field string, pairs []KeyID) {
t.Helper()
importRequest := &pilosa.ImportRequest{
Index: index,
Field: field,
RowIDs: make([]uint64, len(pairs)),
ColumnKeys: make([]string, len(pairs)),
}
for i, pair := range pairs {
importRequest.RowIDs[i] = pair.ID
importRequest.ColumnKeys[i] = pair.Key
}
err := c.Nodes[0].API.Import(context.Background(), nil, importRequest)
if err != nil {
t.Fatalf("importing IDKey data: %v", err)
}
}
// CreateField creates the index (if necessary) and field specified.
func (c *Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field {
t.Helper()

4
test/pilosa.go
View file

@ -196,7 +196,7 @@ func (m *Command) Client() *http.InternalClient {
}
// Query executes a query against the program through the HTTP API.
func (m *Command) Query(t *testing.T, index, rawQuery, query string) (string, error) {
func (m *Command) Query(t testing.TB, index, rawQuery, query string) (string, error) {
resp := Do(t, "POST", fmt.Sprintf("%s/index/%s/query?%s", m.URL(), index, rawQuery), query)
if resp.StatusCode != gohttp.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
@ -285,7 +285,7 @@ func (m *Command) RecalculateCaches(t *testing.T) error {
}
// Do executes http.Do() with an http.NewRequest().
func Do(t *testing.T, method, urlStr string, body string) *httpResponse {
func Do(t testing.TB, method, urlStr string, body string) *httpResponse {
t.Helper()
req, err := gohttp.NewRequest(
method,