scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

change Query and QueryNode to use pilosa.* Query structs

Browse source
This commit is contained in:
Matt Jaffee 2018-07-05 15:02:33 -05:00
parent 59e80f9692
commit e76a90e69b
No known key found for this signature in database
GPG key ID: 08A3DFFF987B11BF
9 changed files with 32 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cache.go
View file

@ -409,14 +409,6 @@ func (p Pairs) String() string {
return buf.String()
}
func EncodePairs(a Pairs) []*internal.Pair {
other := make([]*internal.Pair, len(a))
for i := range a {
other[i] = encodePair(a[i])
}
return other
}
func decodePairs(a []*internal.Pair) []Pair {
other := make([]Pair, len(a))
for i := range a {

14
client.go
View file

@ -3,8 +3,6 @@ package pilosa
import (
"context"
"io"
"github.com/pilosa/pilosa/internal"
)
// Bit represents the intersection of a row and a column. It can be specifed by
@ -35,8 +33,8 @@ type InternalClient interface {
Schema(ctx context.Context) ([]*IndexInfo, error)
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error
ImportK(ctx context.Context, index, field string, bits []Bit) error
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@ -55,12 +53,12 @@ type InternalClient interface {
//===============
type InternalQueryClient interface {
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
}
type NopInternalQueryClient struct{}
func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
@ -90,10 +88,10 @@ func (n NopInternalClient) CreateIndex(ctx context.Context, index string, opt In
func (n NopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) {
return nil, nil
}
func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n NopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error {

6
encoding/proto/proto.go
View file

@ -685,7 +685,11 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) {
func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) {
m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets))
decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets)
m.Err = errors.New(pb.Err)
if pb.Err == "" {
m.Err = nil
} else {
m.Err = errors.New(pb.Err)
}
m.Results = make([]interface{}, len(pb.Results))
decodeQueryResults(pb.Results, m.Results)

47
executor.go
View file

@ -337,7 +337,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "map reduce")
}
// Attach attributes for Row() calls.
@ -1392,7 +1392,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
// exec executes a PQL query remotely for a set of shards on a node.
func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, opt *execOptions) (results []interface{}, err error) {
// Encode request object.
pbreq := &internal.QueryRequest{
pbreq := &QueryRequest{
Query: q.String(),
Shards: shards,
Remote: true,
@ -1403,40 +1403,7 @@ func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *
return nil, err
}
// Return an error, if specified on response.
if err := decodeError(pb.Err); err != nil {
return nil, err
}
// Return appropriate data for the query.
results = make([]interface{}, len(q.Calls))
for i, call := range q.Calls {
var v interface{}
var err error
switch call.Name {
case "Average", "Sum":
v, err = decodeValCount(pb.Results[i].GetValCount()), nil
case "TopN":
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case "Count":
v, err = pb.Results[i].N, nil
case "Set":
v, err = pb.Results[i].Changed, nil
case "Clear":
v, err = pb.Results[i].Changed, nil
case "SetRowAttrs":
case "SetColumnAttrs":
default:
v, err = DecodeRow(pb.Results[i].GetRow()), nil
}
if err != nil {
return nil, err
}
results[i] = v
}
return results, nil
return pb.Results, pb.Err
}
// shardsByNode returns a mapping of nodes to shards.
@ -1490,7 +1457,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
return nil, errors.Wrap(ctx.Err(), "context done")
case resp := <-ch:
// On error retry against remaining nodes. If an error returns then
// the context will cancel and cause all open goroutines to return.
@ -1500,10 +1467,10 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
nodes = Nodes(nodes).Filter(resp.node)
// Begin mapper against secondary nodes.
if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); err == errShardUnavailable {
if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable {
return nil, resp.err
} else if err != nil {
return nil, err
return nil, errors.Wrap(err, "calling mapper")
}
continue
}
@ -1524,7 +1491,7 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
// Group shards together by nodes.
m, err := e.shardsByNode(nodes, index, shards)
if err != nil {
return err
return errors.Wrap(err, "shards by node")
}
// Execute each node in a separate goroutine.

2
executor_test.go
View file

@ -385,7 +385,7 @@ func TestExecutor_Execute_OldPQL(t *testing.T) {
hldr.SetBit("i", "f", 1, 0)
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" {
t.Fatalf("Expected error: 'unknown call: SetBit', got: %v", errors.Cause(err))
t.Fatalf("Expected error: 'unknown call: SetBit', got: %v. Full: %v", errors.Cause(err), err)
}
}

2
fragment.go
View file

@ -1889,7 +1889,7 @@ func (s *fragmentSyncer) syncBlock(id int) error {
}
// Execute query.
queryRequest := &internal.QueryRequest{
queryRequest := &QueryRequest{
Query: buffers[k].String(),
Remote: true,
}

17
http/client.go
View file

@ -220,22 +220,21 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard
}
// Query executes query against the index.
func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
return c.QueryNode(ctx, c.defaultURI, index, queryRequest)
}
// QueryNode executes query against the index, sending the request to the node specified.
func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
if index == "" {
return nil, pilosa.ErrIndexRequired
} else if queryRequest.Query == "" {
return nil, pilosa.ErrQueryRequired
}
// Encode request object.
buf, err := proto.Marshal(queryRequest)
buf, err := c.serializer.Marshal(queryRequest)
if err != nil {
return nil, errors.Wrap(err, "marshaling")
return nil, errors.Wrap(err, "marshaling queryRequest")
}
// Create HTTP request.
@ -265,11 +264,11 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
return nil, errors.New(string(body))
}
qresp := &internal.QueryResponse{}
if err := proto.Unmarshal(body, qresp); err != nil {
qresp := &pilosa.QueryResponse{}
if err := c.serializer.Unmarshal(body, qresp); err != nil {
return nil, fmt.Errorf("unmarshal response: %s", err)
} else if s := qresp.Err; s != "" {
return nil, errors.New(s)
} else if qresp.Err != nil {
return nil, qresp.Err
}
return qresp, nil

9
http/client_test.go
View file

@ -24,7 +24,6 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
@ -131,7 +130,7 @@ func TestClient_MultiNode(t *testing.T) {
client[2] = MustNewClient(c[2].URL(), defaultClient)
topN := 4
queryRequest := &internal.QueryRequest{
queryRequest := &pilosa.QueryRequest{
Query: fmt.Sprintf(`TopN(f, n=%d)`, topN),
Remote: false,
}
@ -147,17 +146,17 @@ func TestClient_MultiNode(t *testing.T) {
}
// Test must return exactly N results.
if len(result.Results[0].Pairs) != topN {
if len(result.Results[0].([]pilosa.Pair)) != topN {
t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
}
p := []*internal.Pair{
p := []pilosa.Pair{
{ID: 100, Count: 12},
{ID: 22, Count: 10},
{ID: 98, Count: 8},
{ID: 99, Count: 7}}
// Valdidate the Top 4 result counts.
if !reflect.DeepEqual(result.Results[0].Pairs, p) {
if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) {
t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
}

50
http/handler.go
View file

@ -1102,56 +1102,6 @@ const (
QueryResultTypeBool
)
func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest {
req := &pilosa.QueryRequest{
Query: pb.Query,
Shards: pb.Shards,
ColumnAttrs: pb.ColumnAttrs,
Remote: pb.Remote,
ExcludeRowAttrs: pb.ExcludeRowAttrs,
ExcludeColumns: pb.ExcludeColumns,
}
return req
}
func encodeQueryResponse(resp *pilosa.QueryResponse) *internal.QueryResponse {
pb := &internal.QueryResponse{
Results: make([]*internal.QueryResult, len(resp.Results)),
ColumnAttrSets: pilosa.EncodeColumnAttrSets(resp.ColumnAttrSets),
}
for i := range resp.Results {
pb.Results[i] = &internal.QueryResult{}
switch result := resp.Results[i].(type) {
case *pilosa.Row:
pb.Results[i].Type = QueryResultTypeRow
pb.Results[i].Row = pilosa.EncodeRow(result)
case []pilosa.Pair:
pb.Results[i].Type = QueryResultTypePairs
pb.Results[i].Pairs = pilosa.EncodePairs(result)
case pilosa.ValCount:
pb.Results[i].Type = QueryResultTypeValCount
pb.Results[i].ValCount = pilosa.EncodeValCount(result)
case uint64:
pb.Results[i].Type = QueryResultTypeUint64
pb.Results[i].N = result
case bool:
pb.Results[i].Type = QueryResultTypeBool
pb.Results[i].Changed = result
case nil:
pb.Results[i].Type = QueryResultTypeNil
}
}
if resp.Err != nil {
pb.Err = resp.Err.Error()
}
return pb
}
// parseUint64Slice returns a slice of uint64s from a comma-delimited string.
func parseUint64Slice(s string) ([]uint64, error) {
var a []uint64