scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 00:55:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #63 from travisturner/row-field-label

Browse source 
Wrap return types: RowIdentifiers, Pair, and []Pair
This commit is contained in:
Travis Turner 2019-12-16 07:42:20 -06:00 • committed by GitHub
commit 6f556fb880
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 1121 additions and 366 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile-clustertests
View file

@ -1,7 +1,7 @@
# This Dockerfile is used for cluster testing - it produces a much larger image
# and includes all of Go as well as some utilities.
FROM golang:1.11
FROM golang:1.13
LABEL maintainer "dev@pilosa.com"

25
cache.go
View file

@ -16,6 +16,7 @@ package pilosa
import (
"bytes"
"encoding/json"
"fmt"
"io"
"sort"
@ -322,6 +323,18 @@ type Pair struct {
Count uint64 `json:"count"`
}
// PairField
type PairField struct {
Pair Pair
Field string
}
// MarshalJSON marshals PairField into a JSON-encoded byte slice,
// excluding `Field`.
func (p PairField) MarshalJSON() ([]byte, error) {
return json.Marshal(p.Pair)
}
// Pairs is a sortable slice of Pair objects.
type Pairs []Pair
@ -397,6 +410,18 @@ func (p Pairs) String() string {
return buf.String()
}
// PairsField
type PairsField struct {
Pairs []Pair
Field string
}
// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
// excluding `Field`.
func (p PairsField) MarshalJSON() ([]byte, error) {
return json.Marshal(p.Pairs)
}
// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64

56
encoding/proto/proto.go
View file

@ -450,6 +450,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
case []pilosa.Pair:
pb.Results[i].Type = queryResultTypePairs
pb.Results[i].Pairs = encodePairs(result)
case *pilosa.PairsField:
pb.Results[i].Type = queryResultTypePairsField
pb.Results[i].PairsField = encodePairsField(result)
case pilosa.ValCount:
pb.Results[i].Type = queryResultTypeValCount
pb.Results[i].ValCount = encodeValCount(result)
@ -471,6 +474,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
case pilosa.Pair:
pb.Results[i].Type = queryResultTypePair
pb.Results[i].Pairs = []*internal.Pair{encodePair(result)}
case pilosa.PairField:
pb.Results[i].Type = queryResultTypePairField
pb.Results[i].Pairs = []*internal.Pair{encodePairField(result)}
case nil:
pb.Results[i].Type = queryResultTypeNil
default:
@ -1101,6 +1107,7 @@ const (
queryResultTypeNil uint32 = iota
queryResultTypeRow
queryResultTypePairs
queryResultTypePairsField
queryResultTypeValCount
queryResultTypeUint64
queryResultTypeBool
@ -1108,6 +1115,7 @@ const (
queryResultTypeGroupCounts
queryResultTypeRowIdentifiers
queryResultTypePair
queryResultTypePairField
queryResultTypeSignedRow
)
@ -1119,6 +1127,8 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} {
return decodeRow(pb.Row)
case queryResultTypePairs:
return decodePairs(pb.Pairs)
case queryResultTypePairsField:
return decodePairsField(pb.PairsField)
case queryResultTypeValCount:
return decodeValCount(pb.ValCount)
case queryResultTypeUint64:
@ -1135,6 +1145,8 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} {
return decodeGroupCounts(pb.GroupCounts)
case queryResultTypePair:
return decodePair(pb.Pairs[0])
case queryResultTypePairField:
return decodePairField(pb.Pairs[0])
}
panic(fmt.Sprintf("unknown type: %d", pb.Type))
}
@ -1243,6 +1255,17 @@ func decodePairs(a []*internal.Pair) []pilosa.Pair {
return other
}
func decodePairsField(a *internal.PairsField) *pilosa.PairsField {
other := &pilosa.PairsField{
Pairs: make([]pilosa.Pair, len(a.Pairs)),
}
for i := range a.Pairs {
other.Pairs[i] = decodePair(a.Pairs[i])
}
other.Field = a.Field
return other
}
func decodePair(pb *internal.Pair) pilosa.Pair {
return pilosa.Pair{
ID: pb.ID,
@ -1251,6 +1274,17 @@ func decodePair(pb *internal.Pair) pilosa.Pair {
}
}
func decodePairField(pb *internal.Pair) pilosa.PairField {
return pilosa.PairField{
Pair: pilosa.Pair{
ID: pb.ID,
Key: pb.Key,
Count: pb.Count,
},
//Field: pb.Field, // TODO: in order to have this, we need PairField in QueryResponse.
}
}
func decodeValCount(pb *internal.ValCount) pilosa.ValCount {
return pilosa.ValCount{
Val: pb.Val,
@ -1346,6 +1380,17 @@ func encodePairs(a pilosa.Pairs) []*internal.Pair {
return other
}
func encodePairsField(a *pilosa.PairsField) *internal.PairsField {
other := &internal.PairsField{
Pairs: make([]*internal.Pair, len(a.Pairs)),
}
for i := range a.Pairs {
other.Pairs[i] = encodePair(a.Pairs[i])
}
other.Field = a.Field
return other
}
func encodePair(p pilosa.Pair) *internal.Pair {
return &internal.Pair{
ID: p.ID,
@ -1354,6 +1399,17 @@ func encodePair(p pilosa.Pair) *internal.Pair {
}
}
func encodePairField(p pilosa.PairField) *internal.Pair {
/*
// TODO: in order to have this, we need PairField in QueryResponse.
return &internal.Pair{
Pair: encodePair(p.Pair),
Field: p.Field,
}
*/
return encodePair(p.Pair)
}
func encodeValCount(vc pilosa.ValCount) *internal.ValCount {
return &internal.ValCount{
Val: vc.Val,

177
executor.go
View file

@ -694,7 +694,8 @@ func (e *executor) executeGenericField(ctx context.Context, index string, c *pql
span.LogKV("name", c.Name)
defer span.Finish()
if field := c.Args["field"]; field == "" {
field := c.Args["field"]
if field == "" {
return SignedRow{}, fmt.Errorf("plugin operation %s(): field required", c.Name)
}
@ -714,6 +715,7 @@ func (e *executor) executeGenericField(ctx context.Context, index string, c *pql
return SignedRow{}, err
}
other, _ := result.(SignedRow)
other.field = field.(string)
return other, nil
}
@ -808,14 +810,19 @@ func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call,
reduceFn := func(prev, v interface{}) interface{} {
// if minRowID exists, and if it is smaller than the other one return it.
// otherwise return the minRowID of the one which exists.
prevp, _ := prev.(Pair)
vp, _ := v.(Pair)
if prevp.Count > 0 && vp.Count > 0 {
if prevp.ID < vp.ID {
if prev == nil {
return v
} else if v == nil {
return prev
}
prevp, _ := prev.(PairField)
vp, _ := v.(PairField)
if prevp.Pair.Count > 0 && vp.Pair.Count > 0 {
if prevp.Pair.ID < vp.Pair.ID {
return prevp
}
return vp
} else if prevp.Count > 0 {
} else if prevp.Pair.Count > 0 {
return prevp
}
return vp
@ -824,7 +831,7 @@ func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call,
return e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
}
// executeMinRow executes a MaxRow() call.
// executeMaxRow executes a MaxRow() call.
func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMaxRow")
defer span.Finish()
@ -842,14 +849,19 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call,
reduceFn := func(prev, v interface{}) interface{} {
// if minRowID exists, and if it is smaller than the other one return it.
// otherwise return the minRowID of the one which exists.
prevp, _ := prev.(Pair)
vp, _ := v.(Pair)
if prevp.Count > 0 && vp.Count > 0 {
if prevp.ID > vp.ID {
if prev == nil {
return v
} else if v == nil {
return prev
}
prevp, _ := prev.(PairField)
vp, _ := v.(PairField)
if prevp.Pair.Count > 0 && vp.Pair.Count > 0 {
if prevp.Pair.ID > vp.Pair.ID {
return prevp
}
return vp
} else if prevp.Count > 0 {
} else if prevp.Pair.Count > 0 {
return prevp
}
return vp
@ -1175,12 +1187,12 @@ func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Cal
}
// executeMinRowShard returns the minimum row ID for a shard.
func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (Pair, error) {
func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (PairField, error) {
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return Pair{}, err
return PairField{}, err
}
filter = row
}
@ -1188,28 +1200,31 @@ func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql.
fieldName, _ := c.Args["field"].(string)
field := e.Holder.Field(index, fieldName)
if field == nil {
return Pair{}, nil
return PairField{}, nil
}
fragment := e.Holder.fragment(index, fieldName, viewStandard, shard)
if fragment == nil {
return Pair{}, nil
return PairField{}, nil
}
minRowID, count := fragment.minRow(filter)
return Pair{
ID: minRowID,
Count: count,
return PairField{
Pair: Pair{
ID: minRowID,
Count: count,
},
Field: fieldName,
}, nil
}
// executeMaxRowShard returns the maximum row ID for a shard.
func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (Pair, error) {
func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (PairField, error) {
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return Pair{}, err
return PairField{}, err
}
filter = row
}
@ -1217,25 +1232,28 @@ func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql.
fieldName, _ := c.Args["field"].(string)
field := e.Holder.Field(index, fieldName)
if field == nil {
return Pair{}, nil
return PairField{}, nil
}
fragment := e.Holder.fragment(index, fieldName, viewStandard, shard)
if fragment == nil {
return Pair{}, nil
return PairField{}, nil
}
maxRowID, count := fragment.maxRow(filter)
return Pair{
ID: maxRowID,
Count: count,
return PairField{
Pair: Pair{
ID: maxRowID,
Count: count,
},
Field: fieldName,
}, nil
}
// executeTopN executes a TopN() call.
// This first performs the TopN() to determine the top results and then
// requeries to retrieve the full counts for each of the top results.
func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) {
func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*PairsField, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopN")
defer span.Finish()
@ -1243,6 +1261,8 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s
if err != nil {
return nil, fmt.Errorf("executeTopN: %v", err)
}
fieldName, _ := c.Args["_field"].(string)
n, _, err := c.UintArg("n")
if err != nil {
return nil, fmt.Errorf("executeTopN: %v", err)
@ -1256,13 +1276,16 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s
// If this call is against specific ids, or we didn't get results,
// or we are part of a larger distributed query then don't refetch.
if len(pairs) == 0 || len(idsArg) > 0 || opt.Remote {
return pairs, nil
if len(pairs.Pairs) == 0 || len(idsArg) > 0 || opt.Remote {
return &PairsField{
Pairs: pairs.Pairs,
Field: fieldName,
}, nil
}
// Only the original caller should refetch the full counts.
other := c.Clone()
ids := Pairs(pairs).Keys()
ids := Pairs(pairs.Pairs).Keys()
sort.Sort(uint64Slice(ids))
other.Args["ids"] = ids
@ -1271,13 +1294,17 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s
return nil, errors.Wrap(err, "retrieving full counts")
}
if n != 0 && int(n) < len(trimmedList) {
trimmedList = trimmedList[0:n]
if n != 0 && int(n) < len(trimmedList.Pairs) {
trimmedList.Pairs = trimmedList.Pairs[0:n]
}
return trimmedList, nil
return &PairsField{
Pairs: trimmedList.Pairs,
Field: fieldName,
}, nil
}
func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) {
func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*PairsField, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShards")
defer span.Finish()
@ -1288,24 +1315,31 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.([]Pair)
return Pairs(other).Add(v.([]Pair))
other, _ := prev.(*PairsField)
vpf, _ := v.(*PairsField)
if other == nil {
return vpf
} else if vpf == nil {
return other
}
other.Pairs = Pairs(other.Pairs).Add(vpf.Pairs)
return other
}
other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
}
results, _ := other.([]Pair)
results, _ := other.(*PairsField)
// Sort final merged results.
sort.Sort(Pairs(results))
sort.Sort(Pairs(results.Pairs))
return results, nil
}
// executeTopNShard executes a TopN call for a single shard.
func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]Pair, error) {
func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*PairsField, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShard")
defer span.Finish()
@ -1351,7 +1385,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca
f := e.Holder.fragment(index, fieldName, viewStandard, shard)
if f == nil {
return nil, nil
return &PairsField{}, nil
} else if f.CacheType == CacheTypeNone {
return nil, fmt.Errorf("cannot compute TopN(), field has no cache: %q", fieldName)
}
@ -1363,7 +1397,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca
if tanimotoThreshold > 100 {
return nil, errors.New("Tanimoto Threshold is from 1 to 100 only")
}
return f.top(topOptions{
pairs, err := f.top(topOptions{
N: int(n),
Src: src,
RowIDs: rowIDs,
@ -1372,6 +1406,13 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca
MinThreshold: minThreshold,
TanimotoThreshold: tanimotoThreshold,
})
if err != nil {
return nil, errors.Wrap(err, "getting top")
}
return &PairsField{
Pairs: pairs,
}, nil
}
// executeDifferenceShard executes a difference() call for a local shard.
@ -1405,8 +1446,14 @@ func (e *executor) executeDifferenceShard(ctx context.Context, index string, c *
// Row query which returns `Columns` and `Keys`.
// TODO: Rename this to something better. Anything.
type RowIdentifiers struct {
Rows []uint64 `json:"rows"`
Keys []string `json:"keys,omitempty"`
Rows []uint64 `json:"rows"`
Keys []string `json:"keys,omitempty"`
field string
}
// Field returns the field name associated to the row.
func (r *RowIdentifiers) Field() string {
return r.field
}
// RowIDs is a query return type for just uint64 row ids.
@ -3535,41 +3582,47 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res
return other, nil
}
case Pair:
case PairField:
if fieldName := callArgString(call, "field"); fieldName != "" {
field := idx.Field(fieldName)
if field == nil {
return nil, fmt.Errorf("field %q not found", fieldName)
}
if field.keys() {
key, err := field.translateStore.TranslateID(result.ID)
key, err := field.translateStore.TranslateID(result.Pair.ID)
if err != nil {
return nil, err
}
if call.Name == "MinRow" || call.Name == "MaxRow" {
result.Key = key
result.Pair.Key = key
return result, nil
}
return Pair{Key: key, Count: result.Count}, nil
return PairField{
Pair: Pair{Key: key, Count: result.Pair.Count},
Field: fieldName,
}, nil
}
}
case []Pair:
case *PairsField:
if fieldName := callArgString(call, "_field"); fieldName != "" {
field := idx.Field(fieldName)
if field == nil {
return nil, fmt.Errorf("field %q not found", fieldName)
}
if field.keys() {
other := make([]Pair, len(result))
for i := range result {
key, err := field.translateStore.TranslateID(result[i].ID)
other := make([]Pair, len(result.Pairs))
for i := range result.Pairs {
key, err := field.translateStore.TranslateID(result.Pairs[i].ID)
if err != nil {
return nil, err
}
other[i] = Pair{Key: key, Count: result[i].Count}
other[i] = Pair{Key: key, Count: result.Pairs[i].Count}
}
return other, nil
return &PairsField{
Pairs: other,
Field: fieldName,
}, nil
}
}
@ -3604,13 +3657,15 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res
return other, nil
case RowIDs:
other := RowIdentifiers{}
fieldName := callArgString(call, "_field")
if fieldName == "" {
return nil, ErrFieldNotFound
}
other := RowIdentifiers{
field: fieldName,
}
if field := idx.Field(fieldName); field == nil {
return nil, ErrFieldNotFound
} else if field.keys() {
@ -3723,12 +3778,18 @@ func needsShards(calls []*pql.Call) bool {
// SignedRow represents a signed *Row with two (neg/pos) *Rows.
type SignedRow struct {
Neg *Row `json:"neg"`
Pos *Row `json:"pos"`
Neg *Row `json:"neg"`
Pos *Row `json:"pos"`
field string
}
// Field returns the field name associated to the signed row.
func (s *SignedRow) Field() string {
return s.field
}
func (sr *SignedRow) union(other SignedRow) SignedRow {
ret := SignedRow{&Row{}, &Row{}}
ret := SignedRow{&Row{}, &Row{}, ""}
// merge in sr
if sr != nil {

177
executor_test.go
View file

@ -945,9 +945,12 @@ func TestExecutor_Execute_TopN(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{
{ID: 0, Count: 5},
{ID: 10, Count: 2},
} else if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 0, Count: 5},
{ID: 10, Count: 2},
},
Field: "f",
}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
@ -986,9 +989,12 @@ func TestExecutor_Execute_TopN(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{
{ID: 0, Count: 5},
{ID: 10, Count: 2},
} else if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 0, Count: 5},
{ID: 10, Count: 2},
},
Field: "f",
}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
@ -1027,11 +1033,16 @@ func TestExecutor_Execute_TopN(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{
{Key: "zero", Count: 5},
{Key: "ten", Count: 2},
}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
} else {
if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{
Pairs: []pilosa.Pair{
{Key: "zero", Count: 5},
{Key: "ten", Count: 2},
},
Field: "f",
}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
})
@ -1069,9 +1080,12 @@ func TestExecutor_Execute_TopN(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil {
t.Fatal(err)
} else if diff := cmp.Diff(result.Results, []interface{}{
[]pilosa.Pair{
{Key: "foo", Count: 5},
{Key: "bar", Count: 2},
&pilosa.PairsField{
Pairs: []pilosa.Pair{
{Key: "foo", Count: 5},
{Key: "bar", Count: 2},
},
Field: "f",
},
}); diff != "" {
t.Fatal(diff)
@ -1154,8 +1168,11 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
// Execute query.
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 4},
} else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 0, Count: 4},
},
Field: "f",
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
@ -1188,8 +1205,11 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
// Execute query.
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 5},
} else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 0, Count: 5},
},
Field: "f",
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
@ -1224,10 +1244,13 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
// Execute query.
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(other=100), n=3)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{
{ID: 20, Count: 3},
{ID: 10, Count: 2},
{ID: 0, Count: 1},
} else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 20, Count: 3},
{ID: 10, Count: 2},
{ID: 0, Count: 1},
},
Field: "f",
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
@ -1247,8 +1270,11 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
}
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1, attrName="category", attrValues=[123])`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
} else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 10, Count: 1},
},
Field: "f",
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
@ -1270,8 +1296,11 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
}
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
} else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 10, Count: 1},
},
Field: "f",
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
@ -1465,7 +1494,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) {
if err != nil {
t.Fatal(err)
}
target := pilosa.Pair{ID: 1, Count: 1}
target := pilosa.PairField{
Pair: pilosa.Pair{ID: 1, Count: 1},
Field: "f",
}
if !reflect.DeepEqual(target, result.Results[0]) {
t.Fatalf("unexpected result %v != %v", target, result.Results[0])
}
@ -1476,7 +1508,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) {
if err != nil {
t.Fatal(err)
}
target := pilosa.Pair{ID: 10000, Count: 1}
target := pilosa.PairField{
Pair: pilosa.Pair{ID: 10000, Count: 1},
Field: "f",
}
if !reflect.DeepEqual(target, result.Results[0]) {
t.Fatalf("unexpected result %v != %v", target, result.Results[0])
}
@ -1512,7 +1547,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) {
if err != nil {
t.Fatal(err)
}
target := pilosa.Pair{Key: "seven-thousand", ID: 1, Count: 1}
target := pilosa.PairField{
Pair: pilosa.Pair{Key: "seven-thousand", ID: 1, Count: 1},
Field: "f",
}
if !reflect.DeepEqual(target, result.Results[0]) {
t.Fatalf("unexpected result %v != %v", target, result.Results[0])
}
@ -1523,7 +1561,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) {
if err != nil {
t.Fatal(err)
}
target := pilosa.Pair{Key: "five-thousand", ID: 5, Count: 1}
target := pilosa.PairField{
Pair: pilosa.Pair{Key: "five-thousand", ID: 5, Count: 1},
Field: "f",
}
if !reflect.DeepEqual(target, result.Results[0]) {
t.Fatalf("unexpected result %v != %v", target, result.Results[0])
}
@ -2420,7 +2461,6 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
if err != nil {
t.Fatalf("creating field: %v", err)
}
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `
Set(500001, fn=5)
Set(1500001, fn=5)
@ -2432,11 +2472,11 @@ Set(3500003, fn=3)
Set(500001, fn=4)
Set(4500001, fn=4)
`}); err != nil {
t.Fatalf("quuerying remote: %v", err)
t.Fatalf("querying remote: %v", err)
}
err := c[0].API.RecalculateCaches(context.Background())
if err != nil {
t.Fatalf("recalcing caches: %v", err)
t.Fatalf("recalculating caches: %v", err)
}
if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{
@ -2444,10 +2484,13 @@ Set(4500001, fn=4)
Query: `TopN(fn, n=3)`,
}); err != nil {
t.Fatalf("topn querying: %v", err)
} else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{
{ID: 5, Count: 4},
{ID: 3, Count: 3},
{ID: 4, Count: 2},
} else if !reflect.DeepEqual(res.Results, []interface{}{&pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 5, Count: 4},
{ID: 3, Count: 3},
{ID: 4, Count: 2},
},
Field: "fn",
}}) {
t.Fatalf("topn wrong results: %v", res.Results)
}
@ -3039,10 +3082,13 @@ func TestExecutor_Execute_ClearRow(t *testing.T) {
// Check the TopN results.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{
{ID: 1, Count: 7},
{ID: 2, Count: 6},
{ID: 3, Count: 5},
} else if !reflect.DeepEqual(res.Results, []interface{}{&pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 1, Count: 7},
{ID: 2, Count: 6},
{ID: 3, Count: 5},
},
Field: "f",
}}) {
t.Fatalf("topn wrong results: %v", res.Results)
}
@ -3057,9 +3103,12 @@ func TestExecutor_Execute_ClearRow(t *testing.T) {
// Ensure that the cleared row doesn't show up in TopN (i.e. it was removed from the cache).
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{
{ID: 1, Count: 7},
{ID: 3, Count: 5},
} else if !reflect.DeepEqual(res.Results, []interface{}{&pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 1, Count: 7},
{ID: 3, Count: 5},
},
Field: "f",
}}) {
t.Fatalf("topn wrong results: %v", res.Results)
}
@ -3280,30 +3329,40 @@ func TestExecutor_Execute_Rows(t *testing.T) {
})
rows := c.Query(t, "i", `Rows(general)`).Results[0].(pilosa.RowIdentifiers)
if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) {
t.Fatalf("unexpected rows: %+v", rows)
if !reflect.DeepEqual(rows.Rows, []uint64{10, 11, 12, 13}) {
t.Fatalf("unexpected rows: %+v", rows.Rows)
} else if rows.Keys != nil {
t.Fatalf("unexpected keys: %+v", rows.Keys)
}
// backwards compatibility
// TODO: remove at Pilosa 2.0
rows = c.Query(t, "i", `Rows(field=general)`).Results[0].(pilosa.RowIdentifiers)
if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) {
t.Fatalf("unexpected rows: %+v", rows)
if !reflect.DeepEqual(rows.Rows, []uint64{10, 11, 12, 13}) {
t.Fatalf("unexpected rows: %+v", rows.Rows)
} else if rows.Keys != nil {
t.Fatalf("unexpected keys: %+v", rows.Keys)
}
rows = c.Query(t, "i", `Rows(general, limit=2)`).Results[0].(pilosa.RowIdentifiers)
if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) {
t.Fatalf("unexpected rows: %+v", rows)
if !reflect.DeepEqual(rows.Rows, []uint64{10, 11}) {
t.Fatalf("unexpected rows: %+v", rows.Rows)
} else if rows.Keys != nil {
t.Fatalf("unexpected keys: %+v", rows.Keys)
}
rows = c.Query(t, "i", `Rows(general, previous=10,limit=2)`).Results[0].(pilosa.RowIdentifiers)
if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) {
t.Fatalf("unexpected rows: %+v", rows)
if !reflect.DeepEqual(rows.Rows, []uint64{11, 12}) {
t.Fatalf("unexpected rows: %+v", rows.Rows)
} else if rows.Keys != nil {
t.Fatalf("unexpected keys: %+v", rows.Keys)
}
rows = c.Query(t, "i", `Rows(general, column=2)`).Results[0].(pilosa.RowIdentifiers)
if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) {
t.Fatalf("unexpected rows: %+v", rows)
if !reflect.DeepEqual(rows.Rows, []uint64{11, 12}) {
t.Fatalf("unexpected rows: %+v", rows.Rows)
} else if rows.Keys != nil {
t.Fatalf("unexpected keys: %+v", rows.Keys)
}
}
@ -3613,9 +3672,13 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) {
t.Run(fmt.Sprintf("#%d_%s", i, test.q), func(t *testing.T) {
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil {
t.Fatal(err)
} else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(
rows, pilosa.RowIdentifiers{Keys: test.exp}) {
t.Fatalf("\ngot: %+v\nexp: %+v", rows, pilosa.RowIdentifiers{Keys: test.exp})
} else {
rows := res.Results[0].(pilosa.RowIdentifiers)
if !reflect.DeepEqual(rows.Keys, test.exp) {
t.Fatalf("\ngot: %+v\nexp: %+v", rows.Keys, test.exp)
} else if rows.Rows != nil {
t.Fatalf("\ngot: %+v\nexp: nil", rows.Rows)
}
}
})
}

35
http/client_test.go
View file

@ -148,7 +148,8 @@ func TestClient_MultiNode(t *testing.T) {
}
// Test must return exactly N results.
if len(result.Results[0].([]pilosa.Pair)) != topN {
pairsField := result.Results[0].(*pilosa.PairsField)
if len(pairsField.Pairs) != topN {
t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
}
p := []pilosa.Pair{
@ -158,7 +159,7 @@ func TestClient_MultiNode(t *testing.T) {
{ID: 99, Count: 7}}
// Valdidate the Top 4 result counts.
if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) {
if !reflect.DeepEqual(pairsField.Pairs, p) {
t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
}
@ -605,14 +606,14 @@ func TestClient_ImportKeys(t *testing.T) {
Index: "keyed",
Query: "TopN(keyedf)",
})
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
t.Fatalf("unexpected response type %T", resp.Results[0])
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
{Key: "green", Count: 3},
{Key: "blue", Count: 2},
{Key: "purple", Count: 1},
}) {
t.Fatalf("unexpected topn result: %v", pairs)
t.Fatalf("unexpected topn result: %v", pairs.Pairs)
}
})
@ -632,14 +633,14 @@ func TestClient_ImportKeys(t *testing.T) {
Index: "keyed",
Query: "TopN(unkeyedf)",
})
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
t.Fatalf("unexpected response type %T", resp.Results[0])
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
{ID: 1, Count: 3},
{ID: 2, Count: 2},
{ID: 3, Count: 1},
}) {
t.Fatalf("unexpected topn result: %v", pairs)
t.Fatalf("unexpected topn result: %v", pairs.Pairs)
}
})
@ -659,14 +660,14 @@ func TestClient_ImportKeys(t *testing.T) {
Index: "unkeyed",
Query: "TopN(keyedf)",
})
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
t.Fatalf("unexpected response type %T", resp.Results[0])
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
{Key: "green", Count: 3},
{Key: "blue", Count: 2},
{Key: "purple", Count: 1},
}) {
t.Fatalf("unexpected topn result: %v", pairs)
t.Fatalf("unexpected topn result: %v", pairs.Pairs)
}
})
})
@ -704,14 +705,14 @@ func TestClient_ImportKeys(t *testing.T) {
Index: "keyed",
Query: "TopN(keyedf0)",
})
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
t.Fatalf("unexpected response type %T", resp.Results[0])
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
{Key: "green", Count: 3},
{Key: "blue", Count: 2},
{Key: "purple", Count: 1},
}) {
t.Fatalf("unexpected topn result: %v", pairs)
t.Fatalf("unexpected topn result: %v", pairs.Pairs)
}
})
@ -736,14 +737,14 @@ func TestClient_ImportKeys(t *testing.T) {
Index: "keyed",
Query: "TopN(keyedf1)",
})
if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
t.Fatalf("unexpected response type %T", resp.Results[0])
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
{Key: "green", Count: 3},
{Key: "blue", Count: 2},
{Key: "purple", Count: 1},
}) {
t.Fatalf("unexpected topn result: %#v", pairs)
t.Fatalf("unexpected topn result: %#v", pairs.Pairs)
}
})
})

2
internal/clustertests/docker-compose.yml
View file

@ -51,6 +51,6 @@ services:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command:
- "cd /go/src/github.com/pilosa/pilosa/ && go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests"
- "cd /go/src/github.com/pilosa/pilosa/ && go test -v -count=1 github.com/pilosa/pilosa/v2/internal/clustertests"
networks:
pilosanet:

72
internal/private.pb.go
View file

@ -32,7 +32,7 @@ func (m *IndexMeta) Reset() { *m = IndexMeta{} }
func (m *IndexMeta) String() string { return proto.CompactTextString(m) }
func (*IndexMeta) ProtoMessage() {}
func (*IndexMeta) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{0}
return fileDescriptor_private_e6d12fddb5948a73, []int{0}
}
func (m *IndexMeta) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -96,7 +96,7 @@ func (m *FieldOptions) Reset() { *m = FieldOptions{} }
func (m *FieldOptions) String() string { return proto.CompactTextString(m) }
func (*FieldOptions) ProtoMessage() {}
func (*FieldOptions) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{1}
return fileDescriptor_private_e6d12fddb5948a73, []int{1}
}
func (m *FieldOptions) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -213,7 +213,7 @@ func (m *ImportResponse) Reset() { *m = ImportResponse{} }
func (m *ImportResponse) String() string { return proto.CompactTextString(m) }
func (*ImportResponse) ProtoMessage() {}
func (*ImportResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{2}
return fileDescriptor_private_e6d12fddb5948a73, []int{2}
}
func (m *ImportResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -264,7 +264,7 @@ func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} }
func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) }
func (*BlockDataRequest) ProtoMessage() {}
func (*BlockDataRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{3}
return fileDescriptor_private_e6d12fddb5948a73, []int{3}
}
func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -340,7 +340,7 @@ func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} }
func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) }
func (*BlockDataResponse) ProtoMessage() {}
func (*BlockDataResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{4}
return fileDescriptor_private_e6d12fddb5948a73, []int{4}
}
func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -394,7 +394,7 @@ func (m *Cache) Reset() { *m = Cache{} }
func (m *Cache) String() string { return proto.CompactTextString(m) }
func (*Cache) ProtoMessage() {}
func (*Cache) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{5}
return fileDescriptor_private_e6d12fddb5948a73, []int{5}
}
func (m *Cache) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -441,7 +441,7 @@ func (m *MaxShards) Reset() { *m = MaxShards{} }
func (m *MaxShards) String() string { return proto.CompactTextString(m) }
func (*MaxShards) ProtoMessage() {}
func (*MaxShards) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{6}
return fileDescriptor_private_e6d12fddb5948a73, []int{6}
}
func (m *MaxShards) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -490,7 +490,7 @@ func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} }
func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) }
func (*CreateShardMessage) ProtoMessage() {}
func (*CreateShardMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{7}
return fileDescriptor_private_e6d12fddb5948a73, []int{7}
}
func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -551,7 +551,7 @@ func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} }
func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteIndexMessage) ProtoMessage() {}
func (*DeleteIndexMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{8}
return fileDescriptor_private_e6d12fddb5948a73, []int{8}
}
func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -599,7 +599,7 @@ func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} }
func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) }
func (*CreateIndexMessage) ProtoMessage() {}
func (*CreateIndexMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{9}
return fileDescriptor_private_e6d12fddb5948a73, []int{9}
}
func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -655,7 +655,7 @@ func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} }
func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) }
func (*CreateFieldMessage) ProtoMessage() {}
func (*CreateFieldMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{10}
return fileDescriptor_private_e6d12fddb5948a73, []int{10}
}
func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -717,7 +717,7 @@ func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} }
func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteFieldMessage) ProtoMessage() {}
func (*DeleteFieldMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{11}
return fileDescriptor_private_e6d12fddb5948a73, []int{11}
}
func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -773,7 +773,7 @@ func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShar
func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteAvailableShardMessage) ProtoMessage() {}
func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{12}
return fileDescriptor_private_e6d12fddb5948a73, []int{12}
}
func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -836,7 +836,7 @@ func (m *Field) Reset() { *m = Field{} }
func (m *Field) String() string { return proto.CompactTextString(m) }
func (*Field) ProtoMessage() {}
func (*Field) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{13}
return fileDescriptor_private_e6d12fddb5948a73, []int{13}
}
func (m *Field) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -897,7 +897,7 @@ func (m *Schema) Reset() { *m = Schema{} }
func (m *Schema) String() string { return proto.CompactTextString(m) }
func (*Schema) ProtoMessage() {}
func (*Schema) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{14}
return fileDescriptor_private_e6d12fddb5948a73, []int{14}
}
func (m *Schema) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -945,7 +945,7 @@ func (m *Index) Reset() { *m = Index{} }
func (m *Index) String() string { return proto.CompactTextString(m) }
func (*Index) ProtoMessage() {}
func (*Index) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{15}
return fileDescriptor_private_e6d12fddb5948a73, []int{15}
}
func (m *Index) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1001,7 +1001,7 @@ func (m *URI) Reset() { *m = URI{} }
func (m *URI) String() string { return proto.CompactTextString(m) }
func (*URI) ProtoMessage() {}
func (*URI) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{16}
return fileDescriptor_private_e6d12fddb5948a73, []int{16}
}
func (m *URI) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1065,7 +1065,7 @@ func (m *Node) Reset() { *m = Node{} }
func (m *Node) String() string { return proto.CompactTextString(m) }
func (*Node) ProtoMessage() {}
func (*Node) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{17}
return fileDescriptor_private_e6d12fddb5948a73, []int{17}
}
func (m *Node) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1134,7 +1134,7 @@ func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} }
func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) }
func (*NodeStateMessage) ProtoMessage() {}
func (*NodeStateMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{18}
return fileDescriptor_private_e6d12fddb5948a73, []int{18}
}
func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1189,7 +1189,7 @@ func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} }
func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) }
func (*NodeEventMessage) ProtoMessage() {}
func (*NodeEventMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{19}
return fileDescriptor_private_e6d12fddb5948a73, []int{19}
}
func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1245,7 +1245,7 @@ func (m *NodeStatus) Reset() { *m = NodeStatus{} }
func (m *NodeStatus) String() string { return proto.CompactTextString(m) }
func (*NodeStatus) ProtoMessage() {}
func (*NodeStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{20}
return fileDescriptor_private_e6d12fddb5948a73, []int{20}
}
func (m *NodeStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1307,7 +1307,7 @@ func (m *IndexStatus) Reset() { *m = IndexStatus{} }
func (m *IndexStatus) String() string { return proto.CompactTextString(m) }
func (*IndexStatus) ProtoMessage() {}
func (*IndexStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{21}
return fileDescriptor_private_e6d12fddb5948a73, []int{21}
}
func (m *IndexStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1362,7 +1362,7 @@ func (m *FieldStatus) Reset() { *m = FieldStatus{} }
func (m *FieldStatus) String() string { return proto.CompactTextString(m) }
func (*FieldStatus) ProtoMessage() {}
func (*FieldStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{22}
return fileDescriptor_private_e6d12fddb5948a73, []int{22}
}
func (m *FieldStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1418,7 +1418,7 @@ func (m *ClusterStatus) Reset() { *m = ClusterStatus{} }
func (m *ClusterStatus) String() string { return proto.CompactTextString(m) }
func (*ClusterStatus) ProtoMessage() {}
func (*ClusterStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{23}
return fileDescriptor_private_e6d12fddb5948a73, []int{23}
}
func (m *ClusterStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1482,7 +1482,7 @@ func (m *BSIGroup) Reset() { *m = BSIGroup{} }
func (m *BSIGroup) String() string { return proto.CompactTextString(m) }
func (*BSIGroup) ProtoMessage() {}
func (*BSIGroup) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{24}
return fileDescriptor_private_e6d12fddb5948a73, []int{24}
}
func (m *BSIGroup) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1552,7 +1552,7 @@ func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} }
func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) }
func (*CreateViewMessage) ProtoMessage() {}
func (*CreateViewMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{25}
return fileDescriptor_private_e6d12fddb5948a73, []int{25}
}
func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1615,7 +1615,7 @@ func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} }
func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteViewMessage) ProtoMessage() {}
func (*DeleteViewMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{26}
return fileDescriptor_private_e6d12fddb5948a73, []int{26}
}
func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1681,7 +1681,7 @@ func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} }
func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) }
func (*ResizeInstruction) ProtoMessage() {}
func (*ResizeInstruction) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{27}
return fileDescriptor_private_e6d12fddb5948a73, []int{27}
}
func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1767,7 +1767,7 @@ func (m *ResizeSource) Reset() { *m = ResizeSource{} }
func (m *ResizeSource) String() string { return proto.CompactTextString(m) }
func (*ResizeSource) ProtoMessage() {}
func (*ResizeSource) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{28}
return fileDescriptor_private_e6d12fddb5948a73, []int{28}
}
func (m *ResizeSource) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1844,7 +1844,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp
func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) }
func (*ResizeInstructionComplete) ProtoMessage() {}
func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{29}
return fileDescriptor_private_e6d12fddb5948a73, []int{29}
}
func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1905,7 +1905,7 @@ func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} }
func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) }
func (*SetCoordinatorMessage) ProtoMessage() {}
func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{30}
return fileDescriptor_private_e6d12fddb5948a73, []int{30}
}
func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1952,7 +1952,7 @@ func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessa
func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) }
func (*UpdateCoordinatorMessage) ProtoMessage() {}
func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{31}
return fileDescriptor_private_e6d12fddb5948a73, []int{31}
}
func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2000,7 +2000,7 @@ func (m *Topology) Reset() { *m = Topology{} }
func (m *Topology) String() string { return proto.CompactTextString(m) }
func (*Topology) ProtoMessage() {}
func (*Topology) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{32}
return fileDescriptor_private_e6d12fddb5948a73, []int{32}
}
func (m *Topology) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2053,7 +2053,7 @@ func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} }
func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) }
func (*RecalculateCaches) ProtoMessage() {}
func (*RecalculateCaches) Descriptor() ([]byte, []int) {
return fileDescriptor_private_b229d027a4642df7, []int{33}
return fileDescriptor_private_e6d12fddb5948a73, []int{33}
}
func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -8982,9 +8982,9 @@ var (
ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow")
)
func init() { proto.RegisterFile("private.proto", fileDescriptor_private_b229d027a4642df7) }
func init() { proto.RegisterFile("private.proto", fileDescriptor_private_e6d12fddb5948a73) }
var fileDescriptor_private_b229d027a4642df7 = []byte{
var fileDescriptor_private_e6d12fddb5948a73 = []byte{
// 1174 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45,
0x18, 0x66, 0x0f, 0x71, 0xec, 0xdf, 0x71, 0x0e, 0xdb, 0x36, 0x6c, 0x0b, 0x0a, 0x66, 0x54, 0x51,

862
internal/public.pb.go
View file

File diff suppressed because it is too large Load diff

13
internal/public.proto
View file

@ -25,6 +25,16 @@ message Pair {
uint64 Count = 2;
}
message PairField {
Pair Pair = 1;
string Field = 2;
}
message PairsField {
repeated Pair Pairs = 1;
string Field = 2;
}
message FieldRow{
string Field = 1;
uint64 RowID = 2;
@ -88,6 +98,7 @@ message QueryResult {
repeated GroupCount GroupCounts = 8;
RowIdentifiers RowIdentifiers = 9;
SignedRow SignedRow = 10;
PairsField PairsField = 11;
}
message ImportRequest {
@ -137,4 +148,4 @@ message ImportColumnAttrsRequest {
string AttrKey = 3;
repeated string AttrVals = 4;
repeated uint64 ColumnIDs = 5;
}
}

51
server/grpc.go
View file

@ -433,35 +433,35 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo
}
*/
}
case pilosa.Pair:
if r.Key != "" {
case pilosa.PairField:
if r.Pair.Key != "" {
results <- &pb.RowResponse{
Headers: []*pb.ColumnInfo{
{Name: "_id", Datatype: "string"},
{Name: r.Field, Datatype: "string"},
{Name: "count", Datatype: "uint64"},
},
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: r.Key}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Count}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: r.Pair.Key}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.Count}},
},
}
} else {
results <- &pb.RowResponse{
Headers: []*pb.ColumnInfo{
{Name: "_id", Datatype: "uint64"},
{Name: r.Field, Datatype: "uint64"},
{Name: "count", Datatype: "uint64"},
},
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.ID}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Count}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.ID}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.Count}},
},
}
}
case []pilosa.Pair:
case *pilosa.PairsField:
// Determine if the ID has string keys.
var stringKeys bool
if len(r) > 0 {
if r[0].Key != "" {
if len(r.Pairs) > 0 {
if r.Pairs[0].Key != "" {
stringKeys = true
}
}
@ -471,10 +471,10 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo
dtype = "string"
}
ci := []*pb.ColumnInfo{
{Name: "_id", Datatype: dtype},
{Name: r.Field, Datatype: dtype},
{Name: "count", Datatype: "uint64"},
}
for _, pair := range r {
for _, pair := range r.Pairs {
if stringKeys {
results <- &pb.RowResponse{
Headers: ci,
@ -528,7 +528,7 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo
}
case pilosa.RowIdentifiers:
if len(r.Keys) > 0 {
ci := []*pb.ColumnInfo{{Name: "_id", Datatype: "string"}}
ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "string"}}
for _, key := range r.Keys {
results <- &pb.RowResponse{
Headers: ci,
@ -538,7 +538,7 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo
ci = nil
}
} else {
ci := []*pb.ColumnInfo{{Name: "_id", Datatype: "uint64"}}
ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "uint64"}}
for _, id := range r.Rows {
results <- &pb.RowResponse{
Headers: ci,
@ -573,6 +573,27 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: r.Val}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: r.Count}},
}}
case pilosa.SignedRow:
// TODO: address the overflow issue with values outside the int64 range
ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "int64"}}
negs := r.Neg.Columns()
for i := len(negs) - 1; i >= 0; i-- {
results <- &pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: -1 * int64(negs[i])}},
}}
ci = nil
}
for _, id := range r.Pos.Columns() {
results <- &pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: int64(id)}},
}}
ci = nil
}
default:
logger.Printf("unhandled %T\n", r)
breakLoop = true

4
server/grpc_internal_test.go
View file

@ -175,7 +175,7 @@ func TestGRPC(t *testing.T) {
Rows: []uint64{10, 11, 12},
},
[]expHeader{
{"_id", "uint64"},
{"", "uint64"}, // This is blank because we don't expose RowIdentifiers.field, so we have no way to set it for tests.
},
[][]expColumn{
{uint64(10)},
@ -189,7 +189,7 @@ func TestGRPC(t *testing.T) {
Keys: []string{"ten", "eleven", "twelve"},
},
[]expHeader{
{"_id", "string"},
{"", "string"}, // This is blank because we don't expose RowIdentifiers.field, so we have no way to set it for tests.
},
[][]expColumn{
{"ten"},

11
server/handler_test.go
View file

@ -224,7 +224,12 @@ func TestHandler_Endpoints(t *testing.T) {
if err != nil {
t.Fatalf("querying: %v", err)
}
if !reflect.DeepEqual(resp.Results[0], []pilosa.Pair{{Count: 12, ID: 0}}) {
if !reflect.DeepEqual(resp.Results[0], &pilosa.PairsField{
Pairs: []pilosa.Pair{
{Count: 12, ID: 0},
},
Field: "f1",
}) {
t.Fatalf("Unexpected result %v", resp.Results[0])
}
@ -504,8 +509,8 @@ func TestHandler_Endpoints(t *testing.T) {
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if a := resp.Results[0].([]pilosa.Pair); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
} else if a := resp.Results[0].(*pilosa.PairsField); len(a.Pairs) != 2 {
t.Fatalf("unexpected pair length: %d", len(a.Pairs))
}
})