diff --git a/Dockerfile b/Dockerfile index 1bcabbdab..01f3a0341 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ARG MAKE_FLAGS COPY . pilosa -RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a ${BUILD_FLAGS}" ${MAKE_FLAGS} +RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a -mod=vendor ${BUILD_FLAGS}" ${MAKE_FLAGS} FROM alpine:3.9.4 diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests index 53621b9ab..6241ae70d 100644 --- a/Dockerfile-clustertests +++ b/Dockerfile-clustertests @@ -1,17 +1,14 @@ # This Dockerfile is used for cluster testing - it produces a much larger image # and includes all of Go as well as some utilities. -FROM golang:1.11 +FROM golang:1.13 LABEL maintainer "dev@pilosa.com" COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ - && GO111MODULE=on make vendor - -RUN cd /go/src/github.com/pilosa/pilosa \ - && CGO_ENABLED=0 make install FLAGS="-a" + && CGO_ENABLED=0 make install FLAGS="-a -mod=vendor" # download pumba for fault injection ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba diff --git a/Makefile b/Makefile index bee802120..a4726b575 100644 --- a/Makefile +++ b/Makefile @@ -16,12 +16,16 @@ RELEASE_ENABLED = $(subst 0,,$(RELEASE)) BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise) BUILD_TAGS += $(if $(RELEASE_ENABLED),release) BUILD_TAGS += shardwidth$(SHARD_WIDTH) +BUILD_TAGS += $(foreach p,$(PLUGINS),plugin$(p)) define LICENSE_HASH_CODE head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " " endef LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go)) +PLUGINS=distinct export GO111MODULE=on +export GOPRIVATE=github.com/molecula +export PLUGINS # Run tests and compile Pilosa default: test build @@ -85,14 +89,14 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml # running. This will catch changes to internal/clustertests/*.go, but if you # make changes to Pilosa, you'll want to run clustertests-build to rebuild the # pilosa image. -clustertests: +clustertests: vendor docker-compose -f $(DOCKER_COMPOSE) down docker-compose -f $(DOCKER_COMPOSE) build client1 docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 # Like clustertests, but rebuilds all images. -clustertests-build: +clustertests-build: vendor docker-compose -f $(DOCKER_COMPOSE) down docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build @@ -127,12 +131,12 @@ generate-proto-grpc: require-protoc require-protoc-gen-gofast generate: generate-protoc generate-stringer generate-pql # Create Docker image from Dockerfile -docker: +docker: vendor docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" . @echo Created docker image: pilosa:$(VERSION) # Create Docker image from Dockerfile (enterprise) -docker-enterprise: +docker-enterprise: vendor docker build --build-arg MAKE_FLAGS="ENTERPRISE=1" -t "pilosa-enterprise:$(VERSION)" . @echo Created docker image: pilosa-enterprise:$(VERSION) diff --git a/api.go b/api.go index 151de44a9..af468914f 100644 --- a/api.go +++ b/api.go @@ -1188,13 +1188,10 @@ func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsReq bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]} } if err := index.ColumnAttrStore().SetBulkAttrs(bulkAttrs); err != nil { - return err - } - - if err != nil { api.server.logger.Printf("import error: index=%s, shard=%d, len(columns)=%d, err=%s", req.Index, req.Shard, len(req.ColumnIDs), err) + return errors.Wrap(err, "importing column attrs") } - return errors.Wrap(err, "importing column attrs") + return nil } func importExistenceColumns(index *Index, columnIDs []uint64) error { diff --git a/api/client/grpc.go b/api/client/grpc.go index 2d99ea76e..fa9f33b68 100644 --- a/api/client/grpc.go +++ b/api/client/grpc.go @@ -80,7 +80,7 @@ func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.St // Inspect returns a stream of RowResponse for the given index, columns, and filters. // It is intended to mimic something like "select [fields] from table where recordID IN (...)". -func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, fieldFilters []string) (pb.StreamClient, error) { +func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) { if c.conn == nil { return nil, errors.New("client has not established a grpc connection") } @@ -103,7 +103,10 @@ func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint Index: index, Columns: idsOrKeys, FilterFields: fieldFilters, + Limit: limit, + Offset: offset, }) + if err != nil { return nil, errors.Wrap(err, "getting stream") } else if stream == nil { diff --git a/cache.go b/cache.go index 05572fa11..6ff6f954a 100644 --- a/cache.go +++ b/cache.go @@ -16,6 +16,7 @@ package pilosa import ( "bytes" + "encoding/json" "fmt" "io" "sort" @@ -322,6 +323,18 @@ type Pair struct { Count uint64 `json:"count"` } +// PairField +type PairField struct { + Pair Pair + Field string +} + +// MarshalJSON marshals PairField into a JSON-encoded byte slice, +// excluding `Field`. +func (p PairField) MarshalJSON() ([]byte, error) { + return json.Marshal(p.Pair) +} + // Pairs is a sortable slice of Pair objects. type Pairs []Pair @@ -397,6 +410,18 @@ func (p Pairs) String() string { return buf.String() } +// PairsField +type PairsField struct { + Pairs []Pair + Field string +} + +// MarshalJSON marshals PairsField into a JSON-encoded byte slice, +// excluding `Field`. +func (p PairsField) MarshalJSON() ([]byte, error) { + return json.Marshal(p.Pairs) +} + // uint64Slice represents a sortable slice of uint64 numbers. type uint64Slice []uint64 diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 322cca1d1..c1644ed58 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -450,6 +450,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { case []pilosa.Pair: pb.Results[i].Type = queryResultTypePairs pb.Results[i].Pairs = encodePairs(result) + case *pilosa.PairsField: + pb.Results[i].Type = queryResultTypePairsField + pb.Results[i].PairsField = encodePairsField(result) case pilosa.ValCount: pb.Results[i].Type = queryResultTypeValCount pb.Results[i].ValCount = encodeValCount(result) @@ -471,6 +474,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { case pilosa.Pair: pb.Results[i].Type = queryResultTypePair pb.Results[i].Pairs = []*internal.Pair{encodePair(result)} + case pilosa.PairField: + pb.Results[i].Type = queryResultTypePairField + pb.Results[i].Pairs = []*internal.Pair{encodePairField(result)} case nil: pb.Results[i].Type = queryResultTypeNil default: @@ -1101,6 +1107,7 @@ const ( queryResultTypeNil uint32 = iota queryResultTypeRow queryResultTypePairs + queryResultTypePairsField queryResultTypeValCount queryResultTypeUint64 queryResultTypeBool @@ -1108,6 +1115,7 @@ const ( queryResultTypeGroupCounts queryResultTypeRowIdentifiers queryResultTypePair + queryResultTypePairField queryResultTypeSignedRow ) @@ -1119,6 +1127,8 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} { return decodeRow(pb.Row) case queryResultTypePairs: return decodePairs(pb.Pairs) + case queryResultTypePairsField: + return decodePairsField(pb.PairsField) case queryResultTypeValCount: return decodeValCount(pb.ValCount) case queryResultTypeUint64: @@ -1135,6 +1145,8 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} { return decodeGroupCounts(pb.GroupCounts) case queryResultTypePair: return decodePair(pb.Pairs[0]) + case queryResultTypePairField: + return decodePairField(pb.Pairs[0]) } panic(fmt.Sprintf("unknown type: %d", pb.Type)) } @@ -1243,6 +1255,17 @@ func decodePairs(a []*internal.Pair) []pilosa.Pair { return other } +func decodePairsField(a *internal.PairsField) *pilosa.PairsField { + other := &pilosa.PairsField{ + Pairs: make([]pilosa.Pair, len(a.Pairs)), + } + for i := range a.Pairs { + other.Pairs[i] = decodePair(a.Pairs[i]) + } + other.Field = a.Field + return other +} + func decodePair(pb *internal.Pair) pilosa.Pair { return pilosa.Pair{ ID: pb.ID, @@ -1251,6 +1274,17 @@ func decodePair(pb *internal.Pair) pilosa.Pair { } } +func decodePairField(pb *internal.Pair) pilosa.PairField { + return pilosa.PairField{ + Pair: pilosa.Pair{ + ID: pb.ID, + Key: pb.Key, + Count: pb.Count, + }, + //Field: pb.Field, // TODO: in order to have this, we need PairField in QueryResponse. + } +} + func decodeValCount(pb *internal.ValCount) pilosa.ValCount { return pilosa.ValCount{ Val: pb.Val, @@ -1346,6 +1380,17 @@ func encodePairs(a pilosa.Pairs) []*internal.Pair { return other } +func encodePairsField(a *pilosa.PairsField) *internal.PairsField { + other := &internal.PairsField{ + Pairs: make([]*internal.Pair, len(a.Pairs)), + } + for i := range a.Pairs { + other.Pairs[i] = encodePair(a.Pairs[i]) + } + other.Field = a.Field + return other +} + func encodePair(p pilosa.Pair) *internal.Pair { return &internal.Pair{ ID: p.ID, @@ -1354,6 +1399,17 @@ func encodePair(p pilosa.Pair) *internal.Pair { } } +func encodePairField(p pilosa.PairField) *internal.Pair { + /* + // TODO: in order to have this, we need PairField in QueryResponse. + return &internal.Pair{ + Pair: encodePair(p.Pair), + Field: p.Field, + } + */ + return encodePair(p.Pair) +} + func encodeValCount(vc pilosa.ValCount) *internal.ValCount { return &internal.ValCount{ Val: vc.Val, diff --git a/executor.go b/executor.go index d84563a41..33c7c6cd8 100644 --- a/executor.go +++ b/executor.go @@ -23,7 +23,7 @@ import ( "sync" "time" - "github.com/pilosa/pilosa/v2/ext" + "github.com/molecula/ext" "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/shardwidth" @@ -527,6 +527,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeOptionsCall(ctx, index, c, shards, opt) case "IncludesColumn": return e.executeIncludesColumnCall(ctx, index, c, shards, opt) + case "All": + return e.executeAllCall(ctx, index, c, shards, opt) case "Precomputed": return e.executePrecomputedCall(ctx, index, c, shards, opt) default: @@ -635,6 +637,112 @@ func (e *executor) executeIncludesColumnCall(ctx context.Context, index string, return result.(bool), nil } +// executeAllCall executes an All() call. +func (e *executor) executeAllCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { + rslt := NewRow() + + var limit uint64 + var offset uint64 + + if lim, hasLimit, err := c.UintArg("limit"); err != nil { + return nil, errors.Wrap(err, "getting limit") + } else if hasLimit && lim > 0 { + limit = uint64(lim) + } + if off, hasOffset, err := c.UintArg("offset"); err != nil { + return nil, errors.Wrap(err, "getting offset") + } else if hasOffset && off > 0 { + offset = uint64(off) + } + + if limit == 0 { + limit = math.MaxUint64 + } + + // skip tracks the number of records left to be skipped + // in support of getting to the offset. + var skip uint64 = offset + + // got tracks the number of records gotten to that point. + var got uint64 + + for _, shard := range shards { + row, err := e.executeAllCallMapReduce(ctx, index, c, shard, opt) + if err != nil { + return nil, errors.Wrap(err, "executing map reduce on shard") + } + + segCnt := row.Count() + + // If this segment doesn't reach the offset, skip it. + if segCnt <= skip { + skip -= segCnt + continue + } + + // This segment doesn't have enough to finish fulfilling the limit + // (or it has exactly enough). + if segCnt-skip <= limit-got { + if skip == 0 { + rslt.Merge(row) + } else { + cols := row.Columns() + partialRow := NewRow() + for _, bit := range cols[skip:] { + partialRow.SetBit(bit) + } + rslt.Merge(partialRow) + } + got += segCnt - skip + // In the case where this segment exactly fulfills the limit, break. + if got == limit { + break + } + skip = 0 + continue + } + + // This segment has more records than the remaining limit requires. + cols := row.Columns() + partialRow := NewRow() + for _, bit := range cols[skip : skip+limit-got] { + partialRow.SetBit(bit) + } + rslt.Merge(partialRow) + break + } + + return rslt, nil +} + +// executeAllCallMapReduce executes a single shard of the All() call +// using the executor.mapReduce() method. +func (e *executor) executeAllCallMapReduce(ctx context.Context, index string, c *pql.Call, shard uint64, opt *execOptions) (*Row, error) { + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + return e.executeAllCallShard(ctx, index, c, shard) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(*Row) + if other == nil { + other = NewRow() + } + other.Merge(v.(*Row)) + return other + } + + result, err := e.mapReduce(ctx, index, []uint64{shard}, c, opt, mapFn, reduceFn) + if err != nil { + return nil, errors.Wrap(err, "map reduce") + } + + row, _ := result.(*Row) + + return row, nil +} + // executeIncludesColumnCallShard func (e *executor) executeIncludesColumnCallShard(ctx context.Context, index string, c *pql.Call, shard uint64, column uint64) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIncludesColumnCallShard") @@ -694,7 +802,8 @@ func (e *executor) executeGenericField(ctx context.Context, index string, c *pql span.LogKV("name", c.Name) defer span.Finish() - if field := c.Args["field"]; field == "" { + field := c.Args["field"] + if field == "" { return SignedRow{}, fmt.Errorf("plugin operation %s(): field required", c.Name) } @@ -714,6 +823,7 @@ func (e *executor) executeGenericField(ctx context.Context, index string, c *pql return SignedRow{}, err } other, _ := result.(SignedRow) + other.field = field.(string) return other, nil } @@ -808,14 +918,19 @@ func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call, reduceFn := func(prev, v interface{}) interface{} { // if minRowID exists, and if it is smaller than the other one return it. // otherwise return the minRowID of the one which exists. - prevp, _ := prev.(Pair) - vp, _ := v.(Pair) - if prevp.Count > 0 && vp.Count > 0 { - if prevp.ID < vp.ID { + if prev == nil { + return v + } else if v == nil { + return prev + } + prevp, _ := prev.(PairField) + vp, _ := v.(PairField) + if prevp.Pair.Count > 0 && vp.Pair.Count > 0 { + if prevp.Pair.ID < vp.Pair.ID { return prevp } return vp - } else if prevp.Count > 0 { + } else if prevp.Pair.Count > 0 { return prevp } return vp @@ -824,7 +939,7 @@ func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call, return e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) } -// executeMinRow executes a MaxRow() call. +// executeMaxRow executes a MaxRow() call. func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMaxRow") defer span.Finish() @@ -842,14 +957,19 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call, reduceFn := func(prev, v interface{}) interface{} { // if minRowID exists, and if it is smaller than the other one return it. // otherwise return the minRowID of the one which exists. - prevp, _ := prev.(Pair) - vp, _ := v.(Pair) - if prevp.Count > 0 && vp.Count > 0 { - if prevp.ID > vp.ID { + if prev == nil { + return v + } else if v == nil { + return prev + } + prevp, _ := prev.(PairField) + vp, _ := v.(PairField) + if prevp.Pair.Count > 0 && vp.Pair.Count > 0 { + if prevp.Pair.ID > vp.Pair.ID { return prevp } return vp - } else if prevp.Count > 0 { + } else if prevp.Pair.Count > 0 { return prevp } return vp @@ -993,6 +1113,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * return e.executeNotShard(ctx, index, c, shard) case "Shift": return e.executeShiftShard(ctx, index, c, shard) + case "All": // Allow a shard computation to use All() (note, limit/offset not applied) + return e.executeAllCallShard(ctx, index, c, shard) case "Precomputed": return e.executePrecomputedCallShard(ctx, index, c, shard) default: @@ -1175,12 +1297,12 @@ func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Cal } // executeMinRowShard returns the minimum row ID for a shard. -func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (Pair, error) { +func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (PairField, error) { var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { - return Pair{}, err + return PairField{}, err } filter = row } @@ -1188,28 +1310,31 @@ func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql. fieldName, _ := c.Args["field"].(string) field := e.Holder.Field(index, fieldName) if field == nil { - return Pair{}, nil + return PairField{}, nil } fragment := e.Holder.fragment(index, fieldName, viewStandard, shard) if fragment == nil { - return Pair{}, nil + return PairField{}, nil } minRowID, count := fragment.minRow(filter) - return Pair{ - ID: minRowID, - Count: count, + return PairField{ + Pair: Pair{ + ID: minRowID, + Count: count, + }, + Field: fieldName, }, nil } // executeMaxRowShard returns the maximum row ID for a shard. -func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (Pair, error) { +func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (PairField, error) { var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { - return Pair{}, err + return PairField{}, err } filter = row } @@ -1217,25 +1342,28 @@ func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql. fieldName, _ := c.Args["field"].(string) field := e.Holder.Field(index, fieldName) if field == nil { - return Pair{}, nil + return PairField{}, nil } fragment := e.Holder.fragment(index, fieldName, viewStandard, shard) if fragment == nil { - return Pair{}, nil + return PairField{}, nil } maxRowID, count := fragment.maxRow(filter) - return Pair{ - ID: maxRowID, - Count: count, + return PairField{ + Pair: Pair{ + ID: maxRowID, + Count: count, + }, + Field: fieldName, }, nil } // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. -func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) { +func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopN") defer span.Finish() @@ -1243,6 +1371,8 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) } + + fieldName, _ := c.Args["_field"].(string) n, _, err := c.UintArg("n") if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) @@ -1256,13 +1386,16 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s // If this call is against specific ids, or we didn't get results, // or we are part of a larger distributed query then don't refetch. - if len(pairs) == 0 || len(idsArg) > 0 || opt.Remote { - return pairs, nil + if len(pairs.Pairs) == 0 || len(idsArg) > 0 || opt.Remote { + return &PairsField{ + Pairs: pairs.Pairs, + Field: fieldName, + }, nil } // Only the original caller should refetch the full counts. other := c.Clone() - ids := Pairs(pairs).Keys() + ids := Pairs(pairs.Pairs).Keys() sort.Sort(uint64Slice(ids)) other.Args["ids"] = ids @@ -1271,13 +1404,17 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s return nil, errors.Wrap(err, "retrieving full counts") } - if n != 0 && int(n) < len(trimmedList) { - trimmedList = trimmedList[0:n] + if n != 0 && int(n) < len(trimmedList.Pairs) { + trimmedList.Pairs = trimmedList.Pairs[0:n] } - return trimmedList, nil + + return &PairsField{ + Pairs: trimmedList.Pairs, + Field: fieldName, + }, nil } -func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) { +func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShards") defer span.Finish() @@ -1288,24 +1425,31 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.([]Pair) - return Pairs(other).Add(v.([]Pair)) + other, _ := prev.(*PairsField) + vpf, _ := v.(*PairsField) + if other == nil { + return vpf + } else if vpf == nil { + return other + } + other.Pairs = Pairs(other.Pairs).Add(vpf.Pairs) + return other } other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return nil, err } - results, _ := other.([]Pair) + results, _ := other.(*PairsField) // Sort final merged results. - sort.Sort(Pairs(results)) + sort.Sort(Pairs(results.Pairs)) return results, nil } // executeTopNShard executes a TopN call for a single shard. -func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]Pair, error) { +func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShard") defer span.Finish() @@ -1351,7 +1495,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca f := e.Holder.fragment(index, fieldName, viewStandard, shard) if f == nil { - return nil, nil + return &PairsField{}, nil } else if f.CacheType == CacheTypeNone { return nil, fmt.Errorf("cannot compute TopN(), field has no cache: %q", fieldName) } @@ -1363,7 +1507,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca if tanimotoThreshold > 100 { return nil, errors.New("Tanimoto Threshold is from 1 to 100 only") } - return f.top(topOptions{ + pairs, err := f.top(topOptions{ N: int(n), Src: src, RowIDs: rowIDs, @@ -1372,6 +1516,13 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca MinThreshold: minThreshold, TanimotoThreshold: tanimotoThreshold, }) + if err != nil { + return nil, errors.Wrap(err, "getting top") + } + + return &PairsField{ + Pairs: pairs, + }, nil } // executeDifferenceShard executes a difference() call for a local shard. @@ -1405,8 +1556,14 @@ func (e *executor) executeDifferenceShard(ctx context.Context, index string, c * // Row query which returns `Columns` and `Keys`. // TODO: Rename this to something better. Anything. type RowIdentifiers struct { - Rows []uint64 `json:"rows"` - Keys []string `json:"keys,omitempty"` + Rows []uint64 `json:"rows"` + Keys []string `json:"keys,omitempty"` + field string +} + +// Field returns the field name associated to the row. +func (r *RowIdentifiers) Field() string { + return r.field } // RowIDs is a query return type for just uint64 row ids. @@ -2334,7 +2491,7 @@ func (e *executor) executePrecomputedCallShard(ctx context.Context, index string return nil, fmt.Errorf("per-shard: missing precomputed values for shard %d", shard) } -// executeNotShard executes a not() call for a local shard. +// executeNotShard executes a Not() call for a local shard. func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeNotShard") defer span.Finish() @@ -2369,6 +2526,34 @@ func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Cal return existenceRow.Difference(row), nil } +// executeAllCallShard executes an All() call for a local shard. +func (e *executor) executeAllCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeAllCallShard") + defer span.Finish() + + if len(c.Children) > 0 { + return nil, errors.New("All() does not accept an input row") + } + + // Make sure the index supports existence tracking. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } else if idx.existenceField() == nil { + return nil, errors.Errorf("index does not support existence tracking: %s", index) + } + + var existenceRow *Row + existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard) + if existenceFrag == nil { + existenceRow = NewRow() + } else { + existenceRow = existenceFrag.row(0) + } + + return existenceRow, nil +} + // executeShiftShard executes a shift() call for a local shard. func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { n, _, err := c.IntArg("n") @@ -3574,41 +3759,47 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return other, nil } - case Pair: + case PairField: if fieldName := callArgString(call, "field"); fieldName != "" { field := idx.Field(fieldName) if field == nil { return nil, fmt.Errorf("field %q not found", fieldName) } if field.keys() { - key, err := field.translateStore.TranslateID(result.ID) + key, err := field.translateStore.TranslateID(result.Pair.ID) if err != nil { return nil, err } if call.Name == "MinRow" || call.Name == "MaxRow" { - result.Key = key + result.Pair.Key = key return result, nil } - return Pair{Key: key, Count: result.Count}, nil + return PairField{ + Pair: Pair{Key: key, Count: result.Pair.Count}, + Field: fieldName, + }, nil } } - case []Pair: + case *PairsField: if fieldName := callArgString(call, "_field"); fieldName != "" { field := idx.Field(fieldName) if field == nil { return nil, fmt.Errorf("field %q not found", fieldName) } if field.keys() { - other := make([]Pair, len(result)) - for i := range result { - key, err := field.translateStore.TranslateID(result[i].ID) + other := make([]Pair, len(result.Pairs)) + for i := range result.Pairs { + key, err := field.translateStore.TranslateID(result.Pairs[i].ID) if err != nil { return nil, err } - other[i] = Pair{Key: key, Count: result[i].Count} + other[i] = Pair{Key: key, Count: result.Pairs[i].Count} } - return other, nil + return &PairsField{ + Pairs: other, + Field: fieldName, + }, nil } } @@ -3643,13 +3834,15 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return other, nil case RowIDs: - other := RowIdentifiers{} - fieldName := callArgString(call, "_field") if fieldName == "" { return nil, ErrFieldNotFound } + other := RowIdentifiers{ + field: fieldName, + } + if field := idx.Field(fieldName); field == nil { return nil, ErrFieldNotFound } else if field.keys() { @@ -3762,12 +3955,18 @@ func needsShards(calls []*pql.Call) bool { // SignedRow represents a signed *Row with two (neg/pos) *Rows. type SignedRow struct { - Neg *Row `json:"neg"` - Pos *Row `json:"pos"` + Neg *Row `json:"neg"` + Pos *Row `json:"pos"` + field string +} + +// Field returns the field name associated to the signed row. +func (s *SignedRow) Field() string { + return s.field } func (sr *SignedRow) union(other SignedRow) SignedRow { - ret := SignedRow{&Row{}, &Row{}} + ret := SignedRow{&Row{}, &Row{}, ""} // merge in sr if sr != nil { @@ -3808,9 +4007,13 @@ func (vc *ValCount) smaller(other ValCount) ValCount { if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) { return other } + extra := int64(0) + if vc.Val == other.Val { + extra += other.Count + } return ValCount{ Val: vc.Val, - Count: vc.Count, + Count: vc.Count + extra, } } @@ -3819,9 +4022,13 @@ func (vc *ValCount) larger(other ValCount) ValCount { if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) { return other } + extra := int64(0) + if vc.Val == other.Val { + extra += other.Count + } return ValCount{ Val: vc.Val, - Count: vc.Count, + Count: vc.Count + extra, } } diff --git a/executor_test.go b/executor_test.go index 212b1eebf..ec8865c8a 100644 --- a/executor_test.go +++ b/executor_test.go @@ -945,9 +945,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 10, Count: 2}, + } else if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 0, Count: 5}, + {ID: 10, Count: 2}, + }, + Field: "f", }) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -986,9 +989,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 10, Count: 2}, + } else if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 0, Count: 5}, + {ID: 10, Count: 2}, + }, + Field: "f", }) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1027,11 +1033,16 @@ func TestExecutor_Execute_TopN(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ - {Key: "zero", Count: 5}, - {Key: "ten", Count: 2}, - }) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } else { + if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {Key: "zero", Count: 5}, + {Key: "ten", Count: 2}, + }, + Field: "f", + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } } }) @@ -1069,9 +1080,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) } else if diff := cmp.Diff(result.Results, []interface{}{ - []pilosa.Pair{ - {Key: "foo", Count: 5}, - {Key: "bar", Count: 2}, + &pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {Key: "foo", Count: 5}, + {Key: "bar", Count: 2}, + }, + Field: "f", }, }); diff != "" { t.Fatal(diff) @@ -1154,8 +1168,11 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { // Execute query. if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ - {ID: 0, Count: 4}, + } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 0, Count: 4}, + }, + Field: "f", }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1188,8 +1205,11 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { // Execute query. if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ - {ID: 0, Count: 5}, + } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 0, Count: 5}, + }, + Field: "f", }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1224,10 +1244,13 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { // Execute query. if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(other=100), n=3)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ - {ID: 20, Count: 3}, - {ID: 10, Count: 2}, - {ID: 0, Count: 1}, + } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 20, Count: 3}, + {ID: 10, Count: 2}, + {ID: 0, Count: 1}, + }, + Field: "f", }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1247,8 +1270,11 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { } if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ - {ID: 10, Count: 1}, + } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 10, Count: 1}, + }, + Field: "f", }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1270,8 +1296,11 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { } if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ - {ID: 10, Count: 1}, + } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 10, Count: 1}, + }, + Field: "f", }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1465,7 +1494,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { if err != nil { t.Fatal(err) } - target := pilosa.Pair{ID: 1, Count: 1} + target := pilosa.PairField{ + Pair: pilosa.Pair{ID: 1, Count: 1}, + Field: "f", + } if !reflect.DeepEqual(target, result.Results[0]) { t.Fatalf("unexpected result %v != %v", target, result.Results[0]) } @@ -1476,7 +1508,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { if err != nil { t.Fatal(err) } - target := pilosa.Pair{ID: 10000, Count: 1} + target := pilosa.PairField{ + Pair: pilosa.Pair{ID: 10000, Count: 1}, + Field: "f", + } if !reflect.DeepEqual(target, result.Results[0]) { t.Fatalf("unexpected result %v != %v", target, result.Results[0]) } @@ -1512,7 +1547,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { if err != nil { t.Fatal(err) } - target := pilosa.Pair{Key: "seven-thousand", ID: 1, Count: 1} + target := pilosa.PairField{ + Pair: pilosa.Pair{Key: "seven-thousand", ID: 1, Count: 1}, + Field: "f", + } if !reflect.DeepEqual(target, result.Results[0]) { t.Fatalf("unexpected result %v != %v", target, result.Results[0]) } @@ -1523,7 +1561,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { if err != nil { t.Fatal(err) } - target := pilosa.Pair{Key: "five-thousand", ID: 5, Count: 1} + target := pilosa.PairField{ + Pair: pilosa.Pair{Key: "five-thousand", ID: 5, Count: 1}, + Field: "f", + } if !reflect.DeepEqual(target, result.Results[0]) { t.Fatalf("unexpected result %v != %v", target, result.Results[0]) } @@ -2420,7 +2461,6 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { if err != nil { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(500001, fn=5) Set(1500001, fn=5) @@ -2432,11 +2472,11 @@ Set(3500003, fn=3) Set(500001, fn=4) Set(4500001, fn=4) `}); err != nil { - t.Fatalf("quuerying remote: %v", err) + t.Fatalf("querying remote: %v", err) } err := c[0].API.RecalculateCaches(context.Background()) if err != nil { - t.Fatalf("recalcing caches: %v", err) + t.Fatalf("recalculating caches: %v", err) } if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ @@ -2444,10 +2484,13 @@ Set(4500001, fn=4) Query: `TopN(fn, n=3)`, }); err != nil { t.Fatalf("topn querying: %v", err) - } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ - {ID: 5, Count: 4}, - {ID: 3, Count: 3}, - {ID: 4, Count: 2}, + } else if !reflect.DeepEqual(res.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 5, Count: 4}, + {ID: 3, Count: 3}, + {ID: 4, Count: 2}, + }, + Field: "fn", }}) { t.Fatalf("topn wrong results: %v", res.Results) } @@ -2855,6 +2898,175 @@ func TestExecutor_Execute_Not(t *testing.T) { }) } +// Ensure an all query can be executed. +func TestExecutor_Execute_All(t *testing.T) { + t.Run("ColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + fld, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Create an import request that sets a full shard, + // plus a couple bits set on either side of it, and + // a final bit set in a fourth shard. + // + // shard0 shard1 shard2 shard3 + // |----------|----------|----------|----------| + // | **|**********|** | * + // + bitCount := ShardWidth + 5 + req := &pilosa.ImportRequest{ + Index: index.Name(), + Field: fld.Name(), + Shard: 0, + RowIDs: make([]uint64, bitCount), + ColumnIDs: make([]uint64, bitCount), + } + for i := 0; i < bitCount-1; i++ { + req.RowIDs[i] = 10 + req.ColumnIDs[i] = uint64(i + ShardWidth - 2) + } + req.RowIDs[bitCount-1] = 10 + req.ColumnIDs[bitCount-1] = uint64((3 * ShardWidth) + 2) + + if err := c[0].API.Import(context.Background(), req); err != nil { + t.Fatal(err) + } + + tests := []struct { + qry string + expCols []uint64 + expCnt uint64 + }{ + {qry: "All()", expCols: req.ColumnIDs, expCnt: uint64(bitCount)}, + {qry: "All(limit=1)", expCols: req.ColumnIDs[:1], expCnt: 1}, + {qry: "All(limit=4)", expCols: req.ColumnIDs[:4], expCnt: 4}, + {qry: "All(limit=4, offset=4)", expCols: req.ColumnIDs[4:8], expCnt: 4}, + {qry: fmt.Sprintf("All(limit=4, offset=%d)", bitCount-5), expCols: req.ColumnIDs[bitCount-5 : bitCount-1], expCnt: 4}, + {qry: fmt.Sprintf("All(limit=1, offset=%d)", bitCount-2), expCols: req.ColumnIDs[bitCount-2 : bitCount-1], expCnt: 1}, + {qry: fmt.Sprintf("All(limit=1, offset=%d)", bitCount-2), expCols: req.ColumnIDs[bitCount-2 : bitCount-1], expCnt: 1}, + {qry: fmt.Sprintf("All(limit=4, offset=%d)", bitCount-2), expCols: req.ColumnIDs[bitCount-2:], expCnt: 2}, + {qry: fmt.Sprintf("All(limit=4, offset=%d)", bitCount+1), expCols: []uint64{}, expCnt: 0}, + {qry: fmt.Sprintf("All(limit=2, offset=%d)", bitCount-3), expCols: req.ColumnIDs[bitCount-3 : bitCount-1], expCnt: 2}, + {qry: fmt.Sprintf("All(limit=2, offset=%d)", bitCount-5), expCols: req.ColumnIDs[bitCount-5 : bitCount-3], expCnt: 2}, + {qry: "All(limit=2, offset=2)", expCols: req.ColumnIDs[2:4], expCnt: 2}, + {qry: "All(limit=1, offset=1)", expCols: req.ColumnIDs[1:2], expCnt: 1}, + {qry: fmt.Sprintf("All(limit=%d, offset=2)", ShardWidth), expCols: req.ColumnIDs[2 : bitCount-3], expCnt: ShardWidth}, + } + for i, test := range tests { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { + t.Fatal(err) + } else if cnt := res.Results[0].(*pilosa.Row).Count(); cnt != test.expCnt { + t.Fatalf("test %d, unexpected count, got: %d, but expected: %d", i, cnt, test.expCnt) + } else if cols := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(cols, test.expCols) { + // If the error results are too large, just show the count. + if len(cols) > 1000 || len(test.expCols) > 1000 { + t.Fatalf("test %d, unexpected columns, got: len(%d), but expected: len(%d)", i, len(cols), len(test.expCols)) + } else { + t.Fatalf("test %d, unexpected columns, got: %v, but expected: %v", i, cols, test.expCols) + } + } + } + }) + + t.Run("ColumnKey", func(t *testing.T) { + c := test.MustRunCluster(t, 1, []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + ), + }) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true, Keys: true}) + fld, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Create an import request that sets key columns + // + // shard0 + // |----------| + // |**** | + // + bitCount := 4 + req := &pilosa.ImportRequest{ + Index: index.Name(), + Field: fld.Name(), + Shard: 0, + RowIDs: make([]uint64, bitCount), + ColumnKeys: make([]string, bitCount), + } + for i := 0; i < bitCount; i++ { + req.RowIDs[i] = 10 + req.ColumnKeys[i] = fmt.Sprintf("c%d", i) + } + + if err := c[0].API.Import(context.Background(), req); err != nil { + t.Fatal(err) + } + + tests := []struct { + qry string + expCols []string + expCnt uint64 + }{ + {qry: "All()", expCols: req.ColumnKeys, expCnt: uint64(bitCount)}, + {qry: "All(limit=1)", expCols: req.ColumnKeys[:1], expCnt: 1}, + {qry: "All(limit=4)", expCols: req.ColumnKeys, expCnt: 4}, + {qry: "All(limit=5)", expCols: req.ColumnKeys, expCnt: 4}, + {qry: "All(limit=1, offset=1)", expCols: req.ColumnKeys[1:2], expCnt: 1}, + {qry: "All(limit=4, offset=1)", expCols: req.ColumnKeys[1:], expCnt: 3}, + {qry: "All(limit=4, offset=5)", expCols: nil, expCnt: 0}, + } + for i, test := range tests { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { + t.Fatal(err) + } else if cnt := len(res.Results[0].(*pilosa.Row).Keys); uint64(cnt) != test.expCnt { + t.Fatalf("test %d, unexpected count, got: %d, but expected: %d", i, cnt, test.expCnt) + } else if cols := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(cols, test.expCols) { + // If the error results are too large, just show the count. + if len(cols) > 1000 || len(test.expCols) > 1000 { + t.Fatalf("test %d, unexpected columns, got: len(%d), but expected: len(%d)", i, len(cols), len(test.expCols)) + } else { + t.Fatalf("test %d, unexpected columns, got: %T, but expected: %T", i, cols, test.expCols) + } + } + } + }) + + // Ensure that a query which uses All() at the shard level can call it. + t.Run("AllShard", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` +Set(3001, f=3) +Set(5001, f=5) +Set(5002, f=5) +`}); err != nil { + t.Fatalf("querying remote: %v", err) + } + + expCols := []uint64{5001, 5002} + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Intersect(All(), Row(f=5))"}); err != nil { + t.Fatal(err) + } else if cols := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(cols, expCols) { + t.Fatalf("unexpected columns, got: %v, but expected: %v", cols, expCols) + } + }) +} + // Ensure a row can be cleared. func TestExecutor_Execute_ClearRow(t *testing.T) { // Set and Mutex tests use the same data and queries @@ -3039,10 +3251,13 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { // Check the TopN results. if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ - {ID: 1, Count: 7}, - {ID: 2, Count: 6}, - {ID: 3, Count: 5}, + } else if !reflect.DeepEqual(res.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 1, Count: 7}, + {ID: 2, Count: 6}, + {ID: 3, Count: 5}, + }, + Field: "f", }}) { t.Fatalf("topn wrong results: %v", res.Results) } @@ -3057,9 +3272,12 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { // Ensure that the cleared row doesn't show up in TopN (i.e. it was removed from the cache). if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ - {ID: 1, Count: 7}, - {ID: 3, Count: 5}, + } else if !reflect.DeepEqual(res.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 1, Count: 7}, + {ID: 3, Count: 5}, + }, + Field: "f", }}) { t.Fatalf("topn wrong results: %v", res.Results) } @@ -3280,30 +3498,40 @@ func TestExecutor_Execute_Rows(t *testing.T) { }) rows := c.Query(t, "i", `Rows(general)`).Results[0].(pilosa.RowIdentifiers) - if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) { - t.Fatalf("unexpected rows: %+v", rows) + if !reflect.DeepEqual(rows.Rows, []uint64{10, 11, 12, 13}) { + t.Fatalf("unexpected rows: %+v", rows.Rows) + } else if rows.Keys != nil { + t.Fatalf("unexpected keys: %+v", rows.Keys) } // backwards compatibility // TODO: remove at Pilosa 2.0 rows = c.Query(t, "i", `Rows(field=general)`).Results[0].(pilosa.RowIdentifiers) - if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) { - t.Fatalf("unexpected rows: %+v", rows) + if !reflect.DeepEqual(rows.Rows, []uint64{10, 11, 12, 13}) { + t.Fatalf("unexpected rows: %+v", rows.Rows) + } else if rows.Keys != nil { + t.Fatalf("unexpected keys: %+v", rows.Keys) } rows = c.Query(t, "i", `Rows(general, limit=2)`).Results[0].(pilosa.RowIdentifiers) - if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) { - t.Fatalf("unexpected rows: %+v", rows) + if !reflect.DeepEqual(rows.Rows, []uint64{10, 11}) { + t.Fatalf("unexpected rows: %+v", rows.Rows) + } else if rows.Keys != nil { + t.Fatalf("unexpected keys: %+v", rows.Keys) } rows = c.Query(t, "i", `Rows(general, previous=10,limit=2)`).Results[0].(pilosa.RowIdentifiers) - if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { - t.Fatalf("unexpected rows: %+v", rows) + if !reflect.DeepEqual(rows.Rows, []uint64{11, 12}) { + t.Fatalf("unexpected rows: %+v", rows.Rows) + } else if rows.Keys != nil { + t.Fatalf("unexpected keys: %+v", rows.Keys) } rows = c.Query(t, "i", `Rows(general, column=2)`).Results[0].(pilosa.RowIdentifiers) - if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { - t.Fatalf("unexpected rows: %+v", rows) + if !reflect.DeepEqual(rows.Rows, []uint64{11, 12}) { + t.Fatalf("unexpected rows: %+v", rows.Rows) + } else if rows.Keys != nil { + t.Fatalf("unexpected keys: %+v", rows.Keys) } } @@ -3613,9 +3841,13 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { t.Run(fmt.Sprintf("#%d_%s", i, test.q), func(t *testing.T) { if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { t.Fatal(err) - } else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual( - rows, pilosa.RowIdentifiers{Keys: test.exp}) { - t.Fatalf("\ngot: %+v\nexp: %+v", rows, pilosa.RowIdentifiers{Keys: test.exp}) + } else { + rows := res.Results[0].(pilosa.RowIdentifiers) + if !reflect.DeepEqual(rows.Keys, test.exp) { + t.Fatalf("\ngot: %+v\nexp: %+v", rows.Keys, test.exp) + } else if rows.Rows != nil { + t.Fatalf("\ngot: %+v\nexp: nil", rows.Rows) + } } }) } @@ -4254,3 +4486,88 @@ func TestExecutor_Execute_IncludesColumn(t *testing.T) { }) }) } + +func TestExecutor_Execute_MinMaxCountEqual(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-1100, 1000)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, f=3) + Set(1, f=3) + Set(2, f=4) + Set(3, f=5) + Set(4, f=5) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=3) + Set(` + strconv.Itoa(ShardWidth+2) + `, f=5) + Set(` + strconv.Itoa(ShardWidth+3) + `, f=5) + Set(` + strconv.Itoa(ShardWidth+4) + `, f=5) + Set(` + strconv.Itoa(ShardWidth+5) + `, f=4) + Set(` + strconv.Itoa(2*ShardWidth+1) + `, f=3) + Set(0, x=3) + Set(1, x=3) + + `}); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 3, cnt: 4}, + {filter: `Row(x=3)`, exp: 3, cnt: 2}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Min(field=f)` + } else { + pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 5, cnt: 5}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Max(field=f)` + } else { + pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) +} diff --git a/ext/ext.go b/ext/ext.go deleted file mode 100644 index b6daa3ef0..000000000 --- a/ext/ext.go +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2019 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package ext provides an EXPERIMENTAL AND TEMPORARY interface to use for -// plugin extensions to Pilosa. DO NOT DEVELOP NEW PLUGINS WITH THIS. The -// replacement design is already in process, but it needs more refinement -// to address issues. This one has those issues, and more. -// -// In the current design, plugins will be loaded at runtime using the -// go `plugin` package, so they should be built as a main package using -// the plugin build mode. -// -// Plugins should not import other packages from Pilosa. -// -// To advertise their functionality, plugins define one or more of a -// handful of symbols which will be checked for at plugin load and used -// to register their functionality. -// -// The plugin interface will check for the following function(s). If the -// functions exist, they must have the given signatures. If they return -// a non-nil error, no ops are registered, and the error message will -// be reported in the Pilosa server's logs. -// -// BitmapOps() ([]BitmapOp, error) -// -// These functions may be absent, and may return nil slices; in either -// case, no ops are registered. -package ext - -// The Bitmap type represents a Pilosa bitmap, and is used for bitmap -// operations. -type Bitmap interface { - // AddN and RemoveN can be used to add or remove values from a bitmap. - AddN(a ...uint64) (int, error) - RemoveN(a ...uint64) (int, error) - - // Lookups - Max() uint64 - Min() (uint64, bool) - Count() uint64 - Any() bool - Contains(uint64) bool - Slice() []uint64 - SliceRange(uint64, uint64) []uint64 - // ContainerBits stores the next 1<<16 bits, starting at the provided - // bit index. It may use a provided []uint64 to store them, or may - // provide its own. Don't write to those bits. Offset must be a multiple - // of 1<<16. - ContainerBits(uint64, []uint64) []uint64 - - // These operators provide existing implemented binary ops. - Intersect(Bitmap) Bitmap - Union(Bitmap) Bitmap - IntersectionCount(Bitmap) uint64 - Difference(Bitmap) Bitmap - Xor(Bitmap) Bitmap - Shift(int) (Bitmap, error) - Flip(uint64, uint64) Bitmap - - // New() is an atrocity: it creates a new bitmap, unrelated to the - // existing bitmap. This lets you create a new bitmap without having - // imported any of the packages that have bitmap creation tools, because - // the bitmap wrapper type has to give you one. - New() Bitmap -} - -// SignedBitmap represents a bitmap that can contain both positive and negative -// values. -type SignedBitmap struct { - Pos, Neg Bitmap -} - -// A BitmapOp represents a new bitmap operation that should be exposed -// in PQL. - -type BitmapOpInput byte -type BitmapOpOutput byte -type BitmapOpArity byte -type BitmapOpPrecall byte -type BitmapOpType struct { - Input BitmapOpInput - Arity BitmapOpArity - Output BitmapOpOutput - Precall BitmapOpPrecall -} - -const ( - OpArityUnary = BitmapOpArity(iota) - OpArityBinary - OpArityNary -) - -const ( - // Unary: Exactly one bitmap. - OpInputBitmap = BitmapOpInput(iota) - // The really weird special case used for BSI, where we end up - // needing to do BSI computations. Arguments will be a - // single BitmapBSI, and a []Bitmap for other operands if any. - OpInputNaryBSI -) - -const ( - OpOutputCount = BitmapOpOutput(iota) - OpOutputBitmap - OpOutputSignedBitmap -) - -const ( - OpPrecallNone = BitmapOpPrecall(iota) - OpPrecallGlobal - OpPrecallLocal // unimplemented -) - -// Regardless of arity, non-BSI functions should always take []Bitmap. -type BitmapOpFunc interface { - BitmapOpType() BitmapOpType -} - -// BitmapOpBitmap should actually always be func([]Bitmap) Bitmap, but -// might be different kinds. -type BitmapOpBitmap interface { - BitmapOpArity() BitmapOpArity - BitmapOpFunc() GenericBitmapOpBitmap -} - -// the common underlying type of the other BitmapOpBitmap functions -type GenericBitmapOpBitmap func([]Bitmap, map[string]interface{}) Bitmap - -// BitmapBSI represents the way a single BSI field is passed into a function -// which takes a BSI field. -type BitmapBSI struct { - FieldData Bitmap - ShardWidth uint64 - Offset int64 - Depth uint -} - -type BitmapOpBSIBitmap func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap - -func (b BitmapOpBSIBitmap) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Output: OpOutputSignedBitmap} -} - -type BitmapOpBSIBitmapPrecall func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap - -func (b BitmapOpBSIBitmapPrecall) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Precall: OpPrecallGlobal, Output: OpOutputSignedBitmap} -} - -type BitmapOpUnaryCount func([]Bitmap, map[string]interface{}) int64 - -func (b BitmapOpUnaryCount) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputCount} -} - -type BitmapOpUnaryBitmap func([]Bitmap, map[string]interface{}) Bitmap - -func (b BitmapOpUnaryBitmap) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputBitmap} -} - -func (b BitmapOpUnaryBitmap) BitmapOpArity() BitmapOpArity { - return OpArityUnary -} - -func (b BitmapOpUnaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap { - return GenericBitmapOpBitmap(b) -} - -type BitmapOpBinaryBitmap func([]Bitmap, map[string]interface{}) Bitmap - -func (b BitmapOpBinaryBitmap) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputBitmap, Arity: OpArityBinary, Output: OpOutputBitmap} -} - -func (b BitmapOpBinaryBitmap) BitmapOpArity() BitmapOpArity { - return OpArityBinary -} - -func (b BitmapOpBinaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap { - return GenericBitmapOpBitmap(b) -} - -type BitmapOpNaryBitmap func([]Bitmap, map[string]interface{}) Bitmap - -func (b BitmapOpNaryBitmap) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputBitmap, Arity: OpArityNary, Output: OpOutputBitmap} -} - -func (b BitmapOpNaryBitmap) BitmapOpArity() BitmapOpArity { - return OpArityNary -} - -func (b BitmapOpNaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap { - return GenericBitmapOpBitmap(b) -} - -// BitmapOp represents an operation to be supported in PQL. Operations -// on bitmaps should always take []Bitmap. Operations on InputNaryBSI should -// take a []Bitmap, plus a Bitmap/shard-width/offset/depth. -// -// Reserved is a list of words to treat as reserved words in a prototype. -// This is not currently used but might be later, and I want to have the -// concept handy now. -type BitmapOp struct { - Name string - Func BitmapOpFunc - Reserved []string -} - -// ExtensionInfo tells us about the extension. The ExtensionAPI string -// should be "v0". The version is a human-readable version, use something -// that seems meaningful. Name and Description are reasonably self-explanatory, -// I hope. -// -// Extensions should define a function: -// func ExtensionInfo(extensionAPI string) (*ExtensionInfo, error) -// which reports their extension info if they think they can coexist with that -// API string. -type ExtensionInfo struct { - Name string // Extension name. - Description string // Short description. - Version string // Human-readable version info for extension. - ExtensionAPI string // Extension API version. Should be v0 for now. - License string // License info. - BitmapOps []BitmapOp // List of provided ops. -} diff --git a/ext/samples/.gitignore b/ext/samples/.gitignore deleted file mode 100644 index a63fa2c94..000000000 --- a/ext/samples/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*/*.so diff --git a/ext/samples/some/some.go b/ext/samples/some/some.go deleted file mode 100644 index 07088d255..000000000 --- a/ext/samples/some/some.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2019 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "math/bits" - - "github.com/molecula/apophenia" - "github.com/pilosa/pilosa/v2/ext" -) - -// This could be dynamically generated, but for now it's not. -// nolint:unused,deadcode -var extInfoTemplate = &ext.ExtensionInfo{ - Name: "some", - Description: "some of the bits/all of the bits/none of the bits", - Version: "0.01", - ExtensionAPI: "v0", - License: "unreleased", - BitmapOps: []ext.BitmapOp{ - {Name: "Some", Func: ext.BitmapOpUnaryBitmap(Some), Reserved: []string{"p", "seed"}}, - }, -} - -// ExtensionInfo is the entry point used by the plugin code. -func ExtensionInfo(api string) (*ext.ExtensionInfo, error) { // nolint:unused,deadcode - return extInfoTemplate, nil -} - -const batchSize = 1024 - -// Some returns some of the bits from its first input bitmap. Takes seed (int) -// and p (float) values. Seed defaults to 0. -func Some(inputs []ext.Bitmap, args map[string]interface{}) ext.Bitmap { - if len(inputs) == 0 || inputs[0] == nil { - return nil - } - input := inputs[0] - min, ok := input.Min() - // no bits found? - if !ok { - return nil - } - // start at multiple of 128 not greater than min. - min &^= 127 - max := input.Max() - p, ok := args["p"].(float64) - if !ok { - return nil - } - // no bits or impossible probability range - if p <= 0 || p > 1 { - return nil - } - // every bit - if p == 1 { - return inputs[0] - } - // On failure, we default to 0. - seed, _ := args["seed"].(int64) - densityScale := uint64(256) - density := uint64(p * float64(densityScale)) - for density == 0 { - densityScale <<= 1 - density = uint64(p * float64(densityScale)) - // too small - if densityScale > (1 << 32) { - return nil - } - } - w, err := apophenia.NewWeighted(apophenia.NewSequence(seed)) - if err != nil { - return nil - } - someBits := input.New() - toAdd := make([]uint64, batchSize) - toAddN := 0 - offset := apophenia.OffsetFor(apophenia.SequenceWeighted, 0, 0, 0) - for i := min; i < max; i += 128 { - offset.Lo = i - randomBits := w.Bits(offset, density, densityScale) - bit := uint64(0) - for randomBits.Lo != 0 { - next := uint64(bits.TrailingZeros64(randomBits.Lo) + 1) - randomBits.Lo >>= next - toAdd[toAddN] = next + bit + i - toAddN++ - bit += next - } - bit = 64 - for randomBits.Hi != 0 { - next := uint64(bits.TrailingZeros64(randomBits.Hi) + 1) - randomBits.Hi >>= next - toAdd[toAddN] = next + bit + i - toAddN++ - bit += next - } - if toAddN > (batchSize - 128) { - // ignore error - _, _ = someBits.AddN(toAdd[:toAddN]...) - toAddN = 0 - } - } - if toAddN > 0 { - _, _ = someBits.AddN(toAdd[:toAddN]...) - } - return input.Intersect(someBits) -} - -func main() { - fmt.Printf("this is a plugin module only.\n") -} diff --git a/extension.go b/extension.go index d449f3f0c..1a492cce3 100644 --- a/extension.go +++ b/extension.go @@ -17,7 +17,7 @@ package pilosa import ( "fmt" - "github.com/pilosa/pilosa/v2/ext" + "github.com/molecula/ext" "github.com/pilosa/pilosa/v2/roaring" ) diff --git a/extensions/distinct.go b/extensions/distinct.go new file mode 100644 index 000000000..42f11c90f --- /dev/null +++ b/extensions/distinct.go @@ -0,0 +1,21 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build plugindistinct + +package extensions + +import ( + _ "github.com/molecula/extensions/distinct" +) diff --git a/extensions/dummy.go b/extensions/dummy.go new file mode 100644 index 000000000..cf418edb5 --- /dev/null +++ b/extensions/dummy.go @@ -0,0 +1,18 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This package contains only things which are conditional on build +// tags. + +package extensions diff --git a/go.mod b/go.mod index eb38d9288..9366e18cd 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,8 @@ require ( github.com/gorilla/mux v1.7.0 github.com/hashicorp/memberlist v0.1.3 github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b + github.com/molecula/ext v0.0.0-20191202195653-240f38a75171 + github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.2.0 github.com/pkg/errors v0.8.1 @@ -34,15 +35,17 @@ require ( github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible // indirect + github.com/youtube/vitess v2.1.1+incompatible // indirect go.uber.org/atomic v1.4.0 // indirect golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect - golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect + golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect golang.org/x/text v0.3.2 // indirect google.golang.org/grpc v1.24.0 modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 + vitess.io/vitess v2.1.1+incompatible // indirect ) go 1.13 diff --git a/go.sum b/go.sum index 6607cf98d..1b1b7af2f 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,10 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y= github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s= +github.com/molecula/ext v0.0.0-20191202195653-240f38a75171 h1:4VK7u/RM+54Yaz8aRB9vIaDSnbKi3M0NQYg5tsZvOT4= +github.com/molecula/ext v0.0.0-20191202195653-240f38a75171/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ= +github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 h1:mDB/dicofRVFuRYcCVPk+JBiVKXlfbzMahuqHvrYqu4= +github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4/go.mod h1:QQgN5OFjuBAi4Q2UYVMzfvi4k9yvg/qqC+MNFB4I9JI= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -153,6 +157,8 @@ github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/ github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youtube/vitess v2.1.1+incompatible h1:SE+P7DNX/jw5RHFs5CHRhZQjq402EJFCD33JhzQMdDw= +github.com/youtube/vitess v2.1.1+incompatible/go.mod h1:hpMim5/30F1r+0P8GGtB29d0gWHr0IZ5unS+CG0zMx8= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -210,3 +216,5 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130= +vitess.io/vitess v2.1.1+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0= diff --git a/http/client_test.go b/http/client_test.go index f8743555e..077a11ff4 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -148,7 +148,8 @@ func TestClient_MultiNode(t *testing.T) { } // Test must return exactly N results. - if len(result.Results[0].([]pilosa.Pair)) != topN { + pairsField := result.Results[0].(*pilosa.PairsField) + if len(pairsField.Pairs) != topN { t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result)) } p := []pilosa.Pair{ @@ -158,7 +159,7 @@ func TestClient_MultiNode(t *testing.T) { {ID: 99, Count: 7}} // Valdidate the Top 4 result counts. - if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) { + if !reflect.DeepEqual(pairsField.Pairs, p) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result)) } @@ -605,14 +606,14 @@ func TestClient_ImportKeys(t *testing.T) { Index: "keyed", Query: "TopN(keyedf)", }) - if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok { + if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok { t.Fatalf("unexpected response type %T", resp.Results[0]) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{ {Key: "green", Count: 3}, {Key: "blue", Count: 2}, {Key: "purple", Count: 1}, }) { - t.Fatalf("unexpected topn result: %v", pairs) + t.Fatalf("unexpected topn result: %v", pairs.Pairs) } }) @@ -632,14 +633,14 @@ func TestClient_ImportKeys(t *testing.T) { Index: "keyed", Query: "TopN(unkeyedf)", }) - if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok { + if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok { t.Fatalf("unexpected response type %T", resp.Results[0]) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{ {ID: 1, Count: 3}, {ID: 2, Count: 2}, {ID: 3, Count: 1}, }) { - t.Fatalf("unexpected topn result: %v", pairs) + t.Fatalf("unexpected topn result: %v", pairs.Pairs) } }) @@ -659,14 +660,14 @@ func TestClient_ImportKeys(t *testing.T) { Index: "unkeyed", Query: "TopN(keyedf)", }) - if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok { + if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok { t.Fatalf("unexpected response type %T", resp.Results[0]) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{ {Key: "green", Count: 3}, {Key: "blue", Count: 2}, {Key: "purple", Count: 1}, }) { - t.Fatalf("unexpected topn result: %v", pairs) + t.Fatalf("unexpected topn result: %v", pairs.Pairs) } }) }) @@ -704,14 +705,14 @@ func TestClient_ImportKeys(t *testing.T) { Index: "keyed", Query: "TopN(keyedf0)", }) - if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok { + if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok { t.Fatalf("unexpected response type %T", resp.Results[0]) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{ {Key: "green", Count: 3}, {Key: "blue", Count: 2}, {Key: "purple", Count: 1}, }) { - t.Fatalf("unexpected topn result: %v", pairs) + t.Fatalf("unexpected topn result: %v", pairs.Pairs) } }) @@ -736,14 +737,14 @@ func TestClient_ImportKeys(t *testing.T) { Index: "keyed", Query: "TopN(keyedf1)", }) - if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok { + if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok { t.Fatalf("unexpected response type %T", resp.Results[0]) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{ {Key: "green", Count: 3}, {Key: "blue", Count: 2}, {Key: "purple", Count: 1}, }) { - t.Fatalf("unexpected topn result: %#v", pairs) + t.Fatalf("unexpected topn result: %#v", pairs.Pairs) } }) }) diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml index 94a96ad5a..36b418921 100644 --- a/internal/clustertests/docker-compose.yml +++ b/internal/clustertests/docker-compose.yml @@ -51,6 +51,6 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock command: - - "cd /go/src/github.com/pilosa/pilosa/ && go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests" + - "cd /go/src/github.com/pilosa/pilosa/ && go test -mod=vendor -v -count=1 github.com/pilosa/pilosa/v2/internal/clustertests" networks: pilosanet: diff --git a/internal/private.pb.go b/internal/private.pb.go index 755370e74..d42e410cf 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -32,7 +32,7 @@ func (m *IndexMeta) Reset() { *m = IndexMeta{} } func (m *IndexMeta) String() string { return proto.CompactTextString(m) } func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{0} + return fileDescriptor_private_e6d12fddb5948a73, []int{0} } func (m *IndexMeta) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -96,7 +96,7 @@ func (m *FieldOptions) Reset() { *m = FieldOptions{} } func (m *FieldOptions) String() string { return proto.CompactTextString(m) } func (*FieldOptions) ProtoMessage() {} func (*FieldOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{1} + return fileDescriptor_private_e6d12fddb5948a73, []int{1} } func (m *FieldOptions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -213,7 +213,7 @@ func (m *ImportResponse) Reset() { *m = ImportResponse{} } func (m *ImportResponse) String() string { return proto.CompactTextString(m) } func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{2} + return fileDescriptor_private_e6d12fddb5948a73, []int{2} } func (m *ImportResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -264,7 +264,7 @@ func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{3} + return fileDescriptor_private_e6d12fddb5948a73, []int{3} } func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -340,7 +340,7 @@ func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{4} + return fileDescriptor_private_e6d12fddb5948a73, []int{4} } func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -394,7 +394,7 @@ func (m *Cache) Reset() { *m = Cache{} } func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} func (*Cache) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{5} + return fileDescriptor_private_e6d12fddb5948a73, []int{5} } func (m *Cache) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -441,7 +441,7 @@ func (m *MaxShards) Reset() { *m = MaxShards{} } func (m *MaxShards) String() string { return proto.CompactTextString(m) } func (*MaxShards) ProtoMessage() {} func (*MaxShards) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{6} + return fileDescriptor_private_e6d12fddb5948a73, []int{6} } func (m *MaxShards) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -490,7 +490,7 @@ func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } func (*CreateShardMessage) ProtoMessage() {} func (*CreateShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{7} + return fileDescriptor_private_e6d12fddb5948a73, []int{7} } func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -551,7 +551,7 @@ func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } func (*DeleteIndexMessage) ProtoMessage() {} func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{8} + return fileDescriptor_private_e6d12fddb5948a73, []int{8} } func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -599,7 +599,7 @@ func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } func (*CreateIndexMessage) ProtoMessage() {} func (*CreateIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{9} + return fileDescriptor_private_e6d12fddb5948a73, []int{9} } func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -655,7 +655,7 @@ func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } func (*CreateFieldMessage) ProtoMessage() {} func (*CreateFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{10} + return fileDescriptor_private_e6d12fddb5948a73, []int{10} } func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -717,7 +717,7 @@ func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } func (*DeleteFieldMessage) ProtoMessage() {} func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{11} + return fileDescriptor_private_e6d12fddb5948a73, []int{11} } func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -773,7 +773,7 @@ func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShar func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) } func (*DeleteAvailableShardMessage) ProtoMessage() {} func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{12} + return fileDescriptor_private_e6d12fddb5948a73, []int{12} } func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -836,7 +836,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{13} + return fileDescriptor_private_e6d12fddb5948a73, []int{13} } func (m *Field) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -897,7 +897,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{14} + return fileDescriptor_private_e6d12fddb5948a73, []int{14} } func (m *Schema) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -945,7 +945,7 @@ func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} func (*Index) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{15} + return fileDescriptor_private_e6d12fddb5948a73, []int{15} } func (m *Index) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1001,7 +1001,7 @@ func (m *URI) Reset() { *m = URI{} } func (m *URI) String() string { return proto.CompactTextString(m) } func (*URI) ProtoMessage() {} func (*URI) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{16} + return fileDescriptor_private_e6d12fddb5948a73, []int{16} } func (m *URI) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1065,7 +1065,7 @@ func (m *Node) Reset() { *m = Node{} } func (m *Node) String() string { return proto.CompactTextString(m) } func (*Node) ProtoMessage() {} func (*Node) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{17} + return fileDescriptor_private_e6d12fddb5948a73, []int{17} } func (m *Node) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1134,7 +1134,7 @@ func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } func (*NodeStateMessage) ProtoMessage() {} func (*NodeStateMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{18} + return fileDescriptor_private_e6d12fddb5948a73, []int{18} } func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1189,7 +1189,7 @@ func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } func (*NodeEventMessage) ProtoMessage() {} func (*NodeEventMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{19} + return fileDescriptor_private_e6d12fddb5948a73, []int{19} } func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1245,7 +1245,7 @@ func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} func (*NodeStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{20} + return fileDescriptor_private_e6d12fddb5948a73, []int{20} } func (m *NodeStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1307,7 +1307,7 @@ func (m *IndexStatus) Reset() { *m = IndexStatus{} } func (m *IndexStatus) String() string { return proto.CompactTextString(m) } func (*IndexStatus) ProtoMessage() {} func (*IndexStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{21} + return fileDescriptor_private_e6d12fddb5948a73, []int{21} } func (m *IndexStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1362,7 +1362,7 @@ func (m *FieldStatus) Reset() { *m = FieldStatus{} } func (m *FieldStatus) String() string { return proto.CompactTextString(m) } func (*FieldStatus) ProtoMessage() {} func (*FieldStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{22} + return fileDescriptor_private_e6d12fddb5948a73, []int{22} } func (m *FieldStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1418,7 +1418,7 @@ func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} func (*ClusterStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{23} + return fileDescriptor_private_e6d12fddb5948a73, []int{23} } func (m *ClusterStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1482,7 +1482,7 @@ func (m *BSIGroup) Reset() { *m = BSIGroup{} } func (m *BSIGroup) String() string { return proto.CompactTextString(m) } func (*BSIGroup) ProtoMessage() {} func (*BSIGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{24} + return fileDescriptor_private_e6d12fddb5948a73, []int{24} } func (m *BSIGroup) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1552,7 +1552,7 @@ func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} func (*CreateViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{25} + return fileDescriptor_private_e6d12fddb5948a73, []int{25} } func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1615,7 +1615,7 @@ func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} func (*DeleteViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{26} + return fileDescriptor_private_e6d12fddb5948a73, []int{26} } func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1681,7 +1681,7 @@ func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} func (*ResizeInstruction) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{27} + return fileDescriptor_private_e6d12fddb5948a73, []int{27} } func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1767,7 +1767,7 @@ func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} func (*ResizeSource) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{28} + return fileDescriptor_private_e6d12fddb5948a73, []int{28} } func (m *ResizeSource) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1844,7 +1844,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{29} + return fileDescriptor_private_e6d12fddb5948a73, []int{29} } func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1905,7 +1905,7 @@ func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{30} + return fileDescriptor_private_e6d12fddb5948a73, []int{30} } func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1952,7 +1952,7 @@ func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessa func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*UpdateCoordinatorMessage) ProtoMessage() {} func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{31} + return fileDescriptor_private_e6d12fddb5948a73, []int{31} } func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2000,7 +2000,7 @@ func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{32} + return fileDescriptor_private_e6d12fddb5948a73, []int{32} } func (m *Topology) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2053,7 +2053,7 @@ func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} func (*RecalculateCaches) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{33} + return fileDescriptor_private_e6d12fddb5948a73, []int{33} } func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -8982,9 +8982,9 @@ var ( ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("private.proto", fileDescriptor_private_b229d027a4642df7) } +func init() { proto.RegisterFile("private.proto", fileDescriptor_private_e6d12fddb5948a73) } -var fileDescriptor_private_b229d027a4642df7 = []byte{ +var fileDescriptor_private_e6d12fddb5948a73 = []byte{ // 1174 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45, 0x18, 0x66, 0x0f, 0x71, 0xec, 0xdf, 0x71, 0x0e, 0xdb, 0x36, 0x6c, 0x0b, 0x0a, 0x66, 0x54, 0x51, diff --git a/internal/public.pb.go b/internal/public.pb.go index 731ae14c0..08ed42318 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -36,7 +36,7 @@ func (m *Row) Reset() { *m = Row{} } func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{0} + return fileDescriptor_public_48374b395a722341, []int{0} } func (m *Row) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -105,7 +105,7 @@ func (m *SignedRow) Reset() { *m = SignedRow{} } func (m *SignedRow) String() string { return proto.CompactTextString(m) } func (*SignedRow) ProtoMessage() {} func (*SignedRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{1} + return fileDescriptor_public_48374b395a722341, []int{1} } func (m *SignedRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -160,7 +160,7 @@ func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } func (*RowIdentifiers) ProtoMessage() {} func (*RowIdentifiers) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{2} + return fileDescriptor_public_48374b395a722341, []int{2} } func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -216,7 +216,7 @@ func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{3} + return fileDescriptor_public_48374b395a722341, []int{3} } func (m *Pair) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -266,6 +266,116 @@ func (m *Pair) GetCount() uint64 { return 0 } +type PairField struct { + Pair *Pair `protobuf:"bytes,1,opt,name=Pair" json:"Pair,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PairField) Reset() { *m = PairField{} } +func (m *PairField) String() string { return proto.CompactTextString(m) } +func (*PairField) ProtoMessage() {} +func (*PairField) Descriptor() ([]byte, []int) { + return fileDescriptor_public_48374b395a722341, []int{4} +} +func (m *PairField) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PairField) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PairField.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *PairField) XXX_Merge(src proto.Message) { + xxx_messageInfo_PairField.Merge(dst, src) +} +func (m *PairField) XXX_Size() int { + return m.Size() +} +func (m *PairField) XXX_DiscardUnknown() { + xxx_messageInfo_PairField.DiscardUnknown(m) +} + +var xxx_messageInfo_PairField proto.InternalMessageInfo + +func (m *PairField) GetPair() *Pair { + if m != nil { + return m.Pair + } + return nil +} + +func (m *PairField) GetField() string { + if m != nil { + return m.Field + } + return "" +} + +type PairsField struct { + Pairs []*Pair `protobuf:"bytes,1,rep,name=Pairs" json:"Pairs,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PairsField) Reset() { *m = PairsField{} } +func (m *PairsField) String() string { return proto.CompactTextString(m) } +func (*PairsField) ProtoMessage() {} +func (*PairsField) Descriptor() ([]byte, []int) { + return fileDescriptor_public_48374b395a722341, []int{5} +} +func (m *PairsField) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PairsField) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PairsField.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *PairsField) XXX_Merge(src proto.Message) { + xxx_messageInfo_PairsField.Merge(dst, src) +} +func (m *PairsField) XXX_Size() int { + return m.Size() +} +func (m *PairsField) XXX_DiscardUnknown() { + xxx_messageInfo_PairsField.DiscardUnknown(m) +} + +var xxx_messageInfo_PairsField proto.InternalMessageInfo + +func (m *PairsField) GetPairs() []*Pair { + if m != nil { + return m.Pairs + } + return nil +} + +func (m *PairsField) GetField() string { + if m != nil { + return m.Field + } + return "" +} + type FieldRow struct { Field string `protobuf:"bytes,1,opt,name=Field,proto3" json:"Field,omitempty"` RowID uint64 `protobuf:"varint,2,opt,name=RowID,proto3" json:"RowID,omitempty"` @@ -279,7 +389,7 @@ func (m *FieldRow) Reset() { *m = FieldRow{} } func (m *FieldRow) String() string { return proto.CompactTextString(m) } func (*FieldRow) ProtoMessage() {} func (*FieldRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{4} + return fileDescriptor_public_48374b395a722341, []int{6} } func (m *FieldRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -342,7 +452,7 @@ func (m *GroupCount) Reset() { *m = GroupCount{} } func (m *GroupCount) String() string { return proto.CompactTextString(m) } func (*GroupCount) ProtoMessage() {} func (*GroupCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{5} + return fileDescriptor_public_48374b395a722341, []int{7} } func (m *GroupCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -404,7 +514,7 @@ func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} func (*ValCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{6} + return fileDescriptor_public_48374b395a722341, []int{8} } func (m *ValCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -460,7 +570,7 @@ func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} func (*ColumnAttrSet) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{7} + return fileDescriptor_public_48374b395a722341, []int{9} } func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -526,7 +636,7 @@ func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{8} + return fileDescriptor_public_48374b395a722341, []int{10} } func (m *Attr) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -608,7 +718,7 @@ func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} func (*AttrMap) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{9} + return fileDescriptor_public_48374b395a722341, []int{11} } func (m *AttrMap) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -661,7 +771,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{10} + return fileDescriptor_public_48374b395a722341, []int{12} } func (m *QueryRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -752,7 +862,7 @@ func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} func (*QueryResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{11} + return fileDescriptor_public_48374b395a722341, []int{13} } func (m *QueryResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -813,6 +923,7 @@ type QueryResult struct { GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers" json:"RowIdentifiers,omitempty"` SignedRow *SignedRow `protobuf:"bytes,10,opt,name=SignedRow" json:"SignedRow,omitempty"` + PairsField *PairsField `protobuf:"bytes,11,opt,name=PairsField" json:"PairsField,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -822,7 +933,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{12} + return fileDescriptor_public_48374b395a722341, []int{14} } func (m *QueryResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -921,6 +1032,13 @@ func (m *QueryResult) GetSignedRow() *SignedRow { return nil } +func (m *QueryResult) GetPairsField() *PairsField { + if m != nil { + return m.PairsField + } + return nil +} + type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -939,7 +1057,7 @@ func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{13} + return fileDescriptor_public_48374b395a722341, []int{15} } func (m *ImportRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1041,7 +1159,7 @@ func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} func (*ImportValueRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{14} + return fileDescriptor_public_48374b395a722341, []int{16} } func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1132,7 +1250,7 @@ func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } func (*TranslateKeysRequest) ProtoMessage() {} func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{15} + return fileDescriptor_public_48374b395a722341, []int{17} } func (m *TranslateKeysRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1193,7 +1311,7 @@ func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } func (*TranslateKeysResponse) ProtoMessage() {} func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{16} + return fileDescriptor_public_48374b395a722341, []int{18} } func (m *TranslateKeysResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1241,7 +1359,7 @@ func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestVi func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequestView) ProtoMessage() {} func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{17} + return fileDescriptor_public_48374b395a722341, []int{19} } func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1296,7 +1414,7 @@ func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequest) ProtoMessage() {} func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{18} + return fileDescriptor_public_48374b395a722341, []int{20} } func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1354,7 +1472,7 @@ func (m *ImportColumnAttrsRequest) Reset() { *m = ImportColumnAttrsReque func (m *ImportColumnAttrsRequest) String() string { return proto.CompactTextString(m) } func (*ImportColumnAttrsRequest) ProtoMessage() {} func (*ImportColumnAttrsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{19} + return fileDescriptor_public_48374b395a722341, []int{21} } func (m *ImportColumnAttrsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1423,6 +1541,8 @@ func init() { proto.RegisterType((*SignedRow)(nil), "internal.SignedRow") proto.RegisterType((*RowIdentifiers)(nil), "internal.RowIdentifiers") proto.RegisterType((*Pair)(nil), "internal.Pair") + proto.RegisterType((*PairField)(nil), "internal.PairField") + proto.RegisterType((*PairsField)(nil), "internal.PairsField") proto.RegisterType((*FieldRow)(nil), "internal.FieldRow") proto.RegisterType((*GroupCount)(nil), "internal.GroupCount") proto.RegisterType((*ValCount)(nil), "internal.ValCount") @@ -1642,6 +1762,82 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *PairField) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PairField) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Pair != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Pair.Size())) + n7, err := m.Pair.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n7 + } + if len(m.Field) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *PairsField) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PairsField) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Pairs) > 0 { + for _, msg := range m.Pairs { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(m.Field) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func (m *FieldRow) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1912,21 +2108,21 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], m.Query) } if len(m.Shards) > 0 { - dAtA8 := make([]byte, len(m.Shards)*10) - var j7 int + dAtA9 := make([]byte, len(m.Shards)*10) + var j8 int for _, num := range m.Shards { for num >= 1<<7 { - dAtA8[j7] = uint8(uint64(num)&0x7f | 0x80) + dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j7++ + j8++ } - dAtA8[j7] = uint8(num) - j7++ + dAtA9[j8] = uint8(num) + j8++ } dAtA[i] = 0x12 i++ - i = encodeVarintPublic(dAtA, i, uint64(j7)) - i += copy(dAtA[i:], dAtA8[:j7]) + i = encodeVarintPublic(dAtA, i, uint64(j8)) + i += copy(dAtA[i:], dAtA9[:j8]) } if m.ColumnAttrs { dAtA[i] = 0x18 @@ -2056,11 +2252,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPublic(dAtA, i, uint64(m.Row.Size())) - n9, err := m.Row.MarshalTo(dAtA[i:]) + n10, err := m.Row.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n9 + i += n10 } if m.N != 0 { dAtA[i] = 0x10 @@ -2093,11 +2289,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size())) - n10, err := m.ValCount.MarshalTo(dAtA[i:]) + n11, err := m.ValCount.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n11 } if m.Type != 0 { dAtA[i] = 0x30 @@ -2105,21 +2301,21 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Type)) } if len(m.RowIDs) > 0 { - dAtA12 := make([]byte, len(m.RowIDs)*10) - var j11 int + dAtA13 := make([]byte, len(m.RowIDs)*10) + var j12 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80) + dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j11++ + j12++ } - dAtA12[j11] = uint8(num) - j11++ + dAtA13[j12] = uint8(num) + j12++ } dAtA[i] = 0x3a i++ - i = encodeVarintPublic(dAtA, i, uint64(j11)) - i += copy(dAtA[i:], dAtA12[:j11]) + i = encodeVarintPublic(dAtA, i, uint64(j12)) + i += copy(dAtA[i:], dAtA13[:j12]) } if len(m.GroupCounts) > 0 { for _, msg := range m.GroupCounts { @@ -2137,21 +2333,31 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x4a i++ i = encodeVarintPublic(dAtA, i, uint64(m.RowIdentifiers.Size())) - n13, err := m.RowIdentifiers.MarshalTo(dAtA[i:]) + n14, err := m.RowIdentifiers.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n14 } if m.SignedRow != nil { dAtA[i] = 0x52 i++ i = encodeVarintPublic(dAtA, i, uint64(m.SignedRow.Size())) - n14, err := m.SignedRow.MarshalTo(dAtA[i:]) + n15, err := m.SignedRow.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n14 + i += n15 + } + if m.PairsField != nil { + dAtA[i] = 0x5a + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.PairsField.Size())) + n16, err := m.PairsField.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n16 } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) @@ -2192,26 +2398,9 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.RowIDs) > 0 { - dAtA16 := make([]byte, len(m.RowIDs)*10) - var j15 int - for _, num := range m.RowIDs { - for num >= 1<<7 { - dAtA16[j15] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j15++ - } - dAtA16[j15] = uint8(num) - j15++ - } - dAtA[i] = 0x22 - i++ - i = encodeVarintPublic(dAtA, i, uint64(j15)) - i += copy(dAtA[i:], dAtA16[:j15]) - } - if len(m.ColumnIDs) > 0 { - dAtA18 := make([]byte, len(m.ColumnIDs)*10) + dAtA18 := make([]byte, len(m.RowIDs)*10) var j17 int - for _, num := range m.ColumnIDs { + for _, num := range m.RowIDs { for num >= 1<<7 { dAtA18[j17] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2220,16 +2409,15 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { dAtA18[j17] = uint8(num) j17++ } - dAtA[i] = 0x2a + dAtA[i] = 0x22 i++ i = encodeVarintPublic(dAtA, i, uint64(j17)) i += copy(dAtA[i:], dAtA18[:j17]) } - if len(m.Timestamps) > 0 { - dAtA20 := make([]byte, len(m.Timestamps)*10) + if len(m.ColumnIDs) > 0 { + dAtA20 := make([]byte, len(m.ColumnIDs)*10) var j19 int - for _, num1 := range m.Timestamps { - num := uint64(num1) + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA20[j19] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2238,11 +2426,29 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { dAtA20[j19] = uint8(num) j19++ } - dAtA[i] = 0x32 + dAtA[i] = 0x2a i++ i = encodeVarintPublic(dAtA, i, uint64(j19)) i += copy(dAtA[i:], dAtA20[:j19]) } + if len(m.Timestamps) > 0 { + dAtA22 := make([]byte, len(m.Timestamps)*10) + var j21 int + for _, num1 := range m.Timestamps { + num := uint64(num1) + for num >= 1<<7 { + dAtA22[j21] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j21++ + } + dAtA22[j21] = uint8(num) + j21++ + } + dAtA[i] = 0x32 + i++ + i = encodeVarintPublic(dAtA, i, uint64(j21)) + i += copy(dAtA[i:], dAtA22[:j21]) + } if len(m.RowKeys) > 0 { for _, s := range m.RowKeys { dAtA[i] = 0x3a @@ -2312,27 +2518,9 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.ColumnIDs) > 0 { - dAtA22 := make([]byte, len(m.ColumnIDs)*10) - var j21 int - for _, num := range m.ColumnIDs { - for num >= 1<<7 { - dAtA22[j21] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j21++ - } - dAtA22[j21] = uint8(num) - j21++ - } - dAtA[i] = 0x2a - i++ - i = encodeVarintPublic(dAtA, i, uint64(j21)) - i += copy(dAtA[i:], dAtA22[:j21]) - } - if len(m.Values) > 0 { - dAtA24 := make([]byte, len(m.Values)*10) + dAtA24 := make([]byte, len(m.ColumnIDs)*10) var j23 int - for _, num1 := range m.Values { - num := uint64(num1) + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA24[j23] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2341,11 +2529,29 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { dAtA24[j23] = uint8(num) j23++ } - dAtA[i] = 0x32 + dAtA[i] = 0x2a i++ i = encodeVarintPublic(dAtA, i, uint64(j23)) i += copy(dAtA[i:], dAtA24[:j23]) } + if len(m.Values) > 0 { + dAtA26 := make([]byte, len(m.Values)*10) + var j25 int + for _, num1 := range m.Values { + num := uint64(num1) + for num >= 1<<7 { + dAtA26[j25] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j25++ + } + dAtA26[j25] = uint8(num) + j25++ + } + dAtA[i] = 0x32 + i++ + i = encodeVarintPublic(dAtA, i, uint64(j25)) + i += copy(dAtA[i:], dAtA26[:j25]) + } if len(m.ColumnKeys) > 0 { for _, s := range m.ColumnKeys { dAtA[i] = 0x3a @@ -2366,8 +2572,8 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(len(m.FloatValues)*8)) for _, num := range m.FloatValues { - f25 := math.Float64bits(float64(num)) - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f25)) + f27 := math.Float64bits(float64(num)) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f27)) i += 8 } } @@ -2441,21 +2647,21 @@ func (m *TranslateKeysResponse) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.IDs) > 0 { - dAtA27 := make([]byte, len(m.IDs)*10) - var j26 int + dAtA29 := make([]byte, len(m.IDs)*10) + var j28 int for _, num := range m.IDs { for num >= 1<<7 { - dAtA27[j26] = uint8(uint64(num)&0x7f | 0x80) + dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j26++ + j28++ } - dAtA27[j26] = uint8(num) - j26++ + dAtA29[j28] = uint8(num) + j28++ } dAtA[i] = 0x1a i++ - i = encodeVarintPublic(dAtA, i, uint64(j26)) - i += copy(dAtA[i:], dAtA27[:j26]) + i = encodeVarintPublic(dAtA, i, uint64(j28)) + i += copy(dAtA[i:], dAtA29[:j28]) } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) @@ -2587,21 +2793,21 @@ func (m *ImportColumnAttrsRequest) MarshalTo(dAtA []byte) (int, error) { } } if len(m.ColumnIDs) > 0 { - dAtA29 := make([]byte, len(m.ColumnIDs)*10) - var j28 int + dAtA31 := make([]byte, len(m.ColumnIDs)*10) + var j30 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) + dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j28++ + j30++ } - dAtA29[j28] = uint8(num) - j28++ + dAtA31[j30] = uint8(num) + j30++ } dAtA[i] = 0x2a i++ - i = encodeVarintPublic(dAtA, i, uint64(j28)) - i += copy(dAtA[i:], dAtA29[:j28]) + i = encodeVarintPublic(dAtA, i, uint64(j30)) + i += copy(dAtA[i:], dAtA31[:j30]) } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) @@ -2720,6 +2926,48 @@ func (m *Pair) Size() (n int) { return n } +func (m *PairField) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Pair != nil { + l = m.Pair.Size() + n += 1 + l + sovPublic(uint64(l)) + } + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *PairsField) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Pairs) > 0 { + for _, e := range m.Pairs { + l = e.Size() + n += 1 + l + sovPublic(uint64(l)) + } + } + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *FieldRow) Size() (n int) { if m == nil { return 0 @@ -2979,6 +3227,10 @@ func (m *QueryResult) Size() (n int) { l = m.SignedRow.Size() n += 1 + l + sovPublic(uint64(l)) } + if m.PairsField != nil { + l = m.PairsField.Size() + n += 1 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -3825,6 +4077,230 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } return nil } +func (m *PairField) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PairField: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PairField: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Pair", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Pair == nil { + m.Pair = &Pair{} + } + if err := m.Pair.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PairsField) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PairsField: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PairsField: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Pairs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Pairs = append(m.Pairs, &Pair{}) + if err := m.Pairs[len(m.Pairs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *FieldRow) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -5312,6 +5788,39 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PairsField", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.PairsField == nil { + m.PairsField = &PairsField{} + } + if err := m.PairsField.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -6903,72 +7412,75 @@ var ( ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("public.proto", fileDescriptor_public_568b1fcbeadcdcca) } +func init() { proto.RegisterFile("public.proto", fileDescriptor_public_48374b395a722341) } -var fileDescriptor_public_568b1fcbeadcdcca = []byte{ - // 1016 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0x1b, 0x45, - 0x14, 0x66, 0xbc, 0xeb, 0x78, 0x7d, 0x9c, 0x84, 0x6a, 0x48, 0xcb, 0x0a, 0x55, 0xc1, 0x1a, 0x21, - 0xb4, 0xdc, 0xa4, 0x6a, 0x90, 0x50, 0xaf, 0xf8, 0x49, 0x93, 0x22, 0xab, 0xaa, 0x55, 0x8e, 0x23, - 0x73, 0x87, 0xb4, 0xa9, 0xa7, 0xee, 0x4a, 0xeb, 0x1d, 0xb3, 0x3f, 0x6c, 0xf3, 0x00, 0x3c, 0x01, - 0x37, 0x88, 0x27, 0xe0, 0x51, 0xb8, 0x42, 0x3c, 0x02, 0x84, 0xc7, 0xe0, 0x06, 0xcd, 0x99, 0x1d, - 0xcf, 0x7a, 0x9b, 0x04, 0x84, 0xb8, 0x3b, 0xdf, 0x39, 0x33, 0x67, 0xce, 0x37, 0xe7, 0x67, 0x06, - 0x76, 0xd7, 0xd5, 0x45, 0x9a, 0xbc, 0x38, 0x5a, 0xe7, 0xaa, 0x54, 0x3c, 0x48, 0xb2, 0x52, 0xe6, - 0x59, 0x9c, 0x8a, 0x02, 0x3c, 0x54, 0x35, 0x0f, 0x61, 0xf0, 0x58, 0xa5, 0xd5, 0x2a, 0x2b, 0x42, - 0x36, 0xf6, 0x22, 0x1f, 0x2d, 0xe4, 0x1f, 0x40, 0xff, 0x8b, 0xb2, 0xcc, 0x8b, 0xb0, 0x37, 0xf6, - 0xa2, 0xd1, 0xf1, 0xfe, 0x91, 0xdd, 0x7a, 0xa4, 0xd5, 0x68, 0x8c, 0x9c, 0x83, 0xff, 0x54, 0x5e, - 0x16, 0xa1, 0x37, 0xf6, 0xa2, 0x21, 0x92, 0xac, 0x7d, 0xa2, 0x8a, 0xf3, 0x24, 0x5b, 0x86, 0xfe, - 0x98, 0x45, 0xbb, 0x68, 0xa1, 0x78, 0x06, 0xc3, 0x59, 0xb2, 0xcc, 0xe4, 0x42, 0x1f, 0xfd, 0x3e, - 0x78, 0xcf, 0x95, 0x3e, 0x96, 0x45, 0xa3, 0xe3, 0x3d, 0xe7, 0x1e, 0x55, 0x8d, 0xda, 0xa2, 0x17, - 0x4c, 0xe5, 0x32, 0xec, 0x5d, 0xbb, 0x60, 0x2a, 0x97, 0xe2, 0x11, 0xec, 0xa3, 0xaa, 0x27, 0x0b, - 0x99, 0x95, 0xc9, 0xcb, 0x44, 0x9a, 0x70, 0x50, 0xd5, 0x96, 0x0b, 0xc9, 0x9b, 0x10, 0x7b, 0x2e, - 0x44, 0xf1, 0x29, 0xf8, 0xcf, 0xe3, 0x24, 0xe7, 0xfb, 0xd0, 0x9b, 0x9c, 0x52, 0x08, 0x3e, 0xf6, - 0x26, 0xa7, 0xfc, 0x00, 0xfa, 0x8f, 0x55, 0x95, 0x95, 0x74, 0xa8, 0x8f, 0x06, 0xf0, 0x3b, 0xe0, - 0x3d, 0x95, 0x97, 0xa1, 0x37, 0x66, 0xd1, 0x10, 0xb5, 0x28, 0xa6, 0x10, 0x3c, 0x49, 0x64, 0x4a, - 0x3c, 0x0e, 0xa0, 0x4f, 0x32, 0xb9, 0x19, 0xa2, 0x01, 0x5a, 0xab, 0x63, 0x3b, 0xb5, 0x9e, 0x08, - 0xf0, 0x7b, 0xb0, 0x83, 0xaa, 0x76, 0xce, 0x1a, 0x24, 0xbe, 0x01, 0xf8, 0x32, 0x57, 0xd5, 0xda, - 0x9c, 0x17, 0x41, 0x9f, 0x10, 0xd1, 0x18, 0x1d, 0x73, 0x47, 0xdd, 0x1e, 0x8a, 0x66, 0xc1, 0xcd, - 0xf1, 0xce, 0xaa, 0x15, 0x1d, 0xe1, 0xa1, 0x16, 0xc5, 0x31, 0x04, 0xf3, 0x38, 0xdd, 0x58, 0xe7, - 0x71, 0x4a, 0xd1, 0x7a, 0xa8, 0xc5, 0x6d, 0x2f, 0x5e, 0xe3, 0x45, 0x7c, 0x0d, 0x7b, 0xa6, 0x16, - 0x74, 0xa6, 0x67, 0xb2, 0x7c, 0xe3, 0xb2, 0xfe, 0x5d, 0x85, 0xbc, 0x79, 0x79, 0x3f, 0x33, 0xf0, - 0xb5, 0xcd, 0x9a, 0xd8, 0xc6, 0xa4, 0x73, 0x75, 0x7e, 0xb9, 0x96, 0x0d, 0x1d, 0x92, 0xf9, 0x18, - 0x46, 0xb3, 0x52, 0x97, 0xcf, 0x3c, 0x4e, 0x2b, 0xd9, 0x38, 0x6a, 0xab, 0xf8, 0x7b, 0x10, 0x4c, - 0xb2, 0xd2, 0x98, 0x7d, 0xa2, 0xb0, 0xc1, 0xfc, 0x3e, 0x0c, 0x4f, 0x94, 0x4a, 0x8d, 0xb1, 0x3f, - 0x66, 0x51, 0x80, 0x4e, 0xc1, 0x0f, 0x01, 0x9e, 0xa4, 0x2a, 0x6e, 0xf6, 0xee, 0x8c, 0x59, 0xc4, - 0xb0, 0xa5, 0x11, 0x0f, 0x60, 0xa0, 0x23, 0x7d, 0x16, 0xaf, 0x1d, 0x5b, 0x76, 0x0b, 0x5b, 0xf1, - 0x17, 0x83, 0xdd, 0xaf, 0x2a, 0x99, 0x5f, 0xa2, 0xfc, 0xb6, 0x92, 0x45, 0xa9, 0xef, 0x96, 0xb0, - 0xad, 0x0e, 0x02, 0xba, 0x0e, 0x66, 0xaf, 0xe2, 0x7c, 0x61, 0xee, 0xce, 0xc7, 0x06, 0x69, 0xae, - 0xee, 0xce, 0x0b, 0xe2, 0x1a, 0x60, 0x5b, 0x45, 0x15, 0x24, 0x57, 0xaa, 0xb4, 0x64, 0x1a, 0xc4, - 0x23, 0x78, 0xfb, 0xec, 0xf5, 0x8b, 0xb4, 0x5a, 0x48, 0x54, 0xb5, 0xd9, 0xbd, 0x43, 0x0b, 0xba, - 0x6a, 0xfe, 0x21, 0xec, 0x37, 0x2a, 0xdb, 0xf9, 0x03, 0x5a, 0xd8, 0xd1, 0xf2, 0x87, 0xb0, 0x7b, - 0xb6, 0xba, 0x90, 0x8b, 0x85, 0x5c, 0x9c, 0xc6, 0x65, 0x1c, 0x06, 0xc4, 0xbb, 0xd3, 0x87, 0x5b, - 0x4b, 0xc4, 0x0f, 0x0c, 0xf6, 0x1a, 0xf6, 0xc5, 0x5a, 0x65, 0x85, 0xd4, 0x29, 0x3e, 0xcb, 0x73, - 0x9b, 0xe2, 0xb3, 0x3c, 0xe7, 0x0f, 0x60, 0x80, 0xb2, 0xa8, 0xd2, 0xd2, 0xd6, 0xcd, 0x5d, 0xe7, - 0xd1, 0xee, 0xad, 0xd2, 0x12, 0xed, 0x2a, 0xfe, 0x19, 0xec, 0x6f, 0xd5, 0xa1, 0x19, 0x36, 0xa3, - 0xe3, 0x77, 0xdd, 0xbe, 0x2d, 0x3b, 0x76, 0x96, 0x8b, 0xef, 0x3d, 0x18, 0xb5, 0x3c, 0xeb, 0xb9, - 0x82, 0xaa, 0xbe, 0x61, 0xf0, 0xe8, 0x8e, 0xde, 0x05, 0x36, 0x6d, 0x4a, 0x90, 0x4d, 0x75, 0xe2, - 0xf5, 0xac, 0xb0, 0xc7, 0xb6, 0x12, 0xaf, 0xd5, 0x68, 0x8c, 0x34, 0x48, 0x5f, 0xc5, 0xd9, 0x52, - 0x2e, 0xa8, 0x04, 0x03, 0xb4, 0x90, 0x1f, 0xb9, 0xde, 0xa3, 0x9c, 0x6d, 0x35, 0xb4, 0xb5, 0xa0, - 0xeb, 0x4f, 0xdb, 0x03, 0x3a, 0x7d, 0x7b, 0x4d, 0x0f, 0x98, 0xb9, 0x31, 0x39, 0xd5, 0xb9, 0xa2, - 0x7a, 0x31, 0x88, 0x7f, 0x02, 0x23, 0x37, 0x37, 0x8a, 0x26, 0x45, 0x07, 0xce, 0xbd, 0x33, 0x62, - 0x7b, 0x21, 0xff, 0xbc, 0x3b, 0x39, 0xc3, 0x21, 0x45, 0x16, 0x6e, 0xdd, 0x46, 0xcb, 0x8e, 0xdd, - 0x49, 0xfb, 0xb0, 0x35, 0xca, 0x43, 0xa0, 0xcd, 0xef, 0xb8, 0xcd, 0x1b, 0x13, 0xba, 0x55, 0xe2, - 0x0f, 0x06, 0x7b, 0x93, 0xd5, 0x5a, 0xe5, 0x65, 0xab, 0x39, 0x26, 0xd9, 0x42, 0xbe, 0xb6, 0xcd, - 0x41, 0xc0, 0x0d, 0xd4, 0x5e, 0x67, 0xa0, 0x52, 0x93, 0x50, 0x53, 0xf8, 0x68, 0x40, 0xeb, 0x62, - 0xfc, 0xad, 0x8b, 0xb9, 0x0f, 0x43, 0x53, 0x05, 0xda, 0xd4, 0x27, 0x93, 0x53, 0xe8, 0xb6, 0x3f, - 0x4f, 0x56, 0xb2, 0x28, 0xe3, 0xd5, 0x5a, 0xf7, 0x89, 0x17, 0x79, 0xd8, 0xd2, 0x98, 0x17, 0xac, - 0xa6, 0x57, 0x63, 0x40, 0xaf, 0x86, 0x85, 0x7a, 0xa7, 0x71, 0x43, 0xc6, 0x80, 0x8c, 0x2d, 0x8d, - 0xf8, 0x95, 0x01, 0x37, 0x1c, 0x69, 0x80, 0xfc, 0x7f, 0x44, 0x6f, 0x27, 0x74, 0x0f, 0x76, 0xe8, - 0x3c, 0x4b, 0xa6, 0x41, 0x9d, 0x70, 0x07, 0xdd, 0x70, 0xf5, 0xbc, 0x71, 0xd3, 0xce, 0xf0, 0x61, - 0xd8, 0x56, 0x89, 0x39, 0x1c, 0x9c, 0xe7, 0x71, 0x56, 0xa4, 0x71, 0x29, 0xf5, 0x96, 0xff, 0xc2, - 0xe8, 0x9a, 0x4f, 0x82, 0xf8, 0x08, 0xee, 0x76, 0xfc, 0xba, 0x89, 0xa1, 0x29, 0x7a, 0x44, 0x51, - 0x8b, 0xe2, 0x04, 0xc2, 0xa6, 0x6c, 0xcc, 0x37, 0xa2, 0x09, 0x61, 0x9e, 0xc8, 0x5a, 0xbb, 0x9e, - 0xc6, 0x2b, 0xd9, 0x44, 0x41, 0xb2, 0xd6, 0xd1, 0xc0, 0xea, 0xd1, 0xe7, 0x83, 0x64, 0xf1, 0x12, - 0x0e, 0xae, 0xf3, 0x41, 0x4f, 0x5f, 0x2a, 0x63, 0x33, 0xa1, 0x02, 0x34, 0x80, 0x3f, 0x82, 0xfe, - 0x77, 0x89, 0xac, 0xed, 0x84, 0x12, 0xae, 0xb0, 0x6f, 0x0a, 0x04, 0xcd, 0x06, 0xf1, 0x13, 0xb3, - 0xc1, 0xb6, 0x86, 0xf6, 0x3f, 0xde, 0x99, 0xc9, 0x77, 0xf3, 0xfa, 0x9a, 0x7c, 0x87, 0xe6, 0xe5, - 0x71, 0x4f, 0xa7, 0x85, 0xfa, 0xb5, 0xd3, 0xe2, 0x3c, 0x4e, 0x4d, 0xd1, 0x0f, 0x71, 0x83, 0x6f, - 0xaf, 0x92, 0x93, 0x3b, 0xbf, 0x5c, 0x1d, 0xb2, 0xdf, 0xae, 0x0e, 0xd9, 0xef, 0x57, 0x87, 0xec, - 0xc7, 0x3f, 0x0f, 0xdf, 0xba, 0xd8, 0xa1, 0x6f, 0xe1, 0xc7, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, - 0x9d, 0x83, 0xa0, 0x41, 0x26, 0x0a, 0x00, 0x00, +var fileDescriptor_public_48374b395a722341 = []byte{ + // 1065 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdb, 0x6e, 0x1c, 0x45, + 0x13, 0xfe, 0x7b, 0x67, 0xd6, 0xbb, 0x5b, 0x6b, 0xfb, 0x8f, 0x9a, 0x4d, 0x18, 0xa1, 0xc8, 0xac, + 0x5a, 0x11, 0x1a, 0x6e, 0x1c, 0xc5, 0x20, 0x94, 0x2b, 0x0e, 0x8e, 0x1d, 0x58, 0x45, 0x59, 0x85, + 0xb2, 0xb5, 0xdc, 0x21, 0x8d, 0xb3, 0x1d, 0x67, 0xa4, 0xd9, 0x99, 0x65, 0x0e, 0x4c, 0xfc, 0x1c, + 0xdc, 0x20, 0x9e, 0x80, 0x77, 0xe0, 0x05, 0xb8, 0x42, 0x3c, 0x02, 0x98, 0xc7, 0xe0, 0x06, 0x55, + 0xf5, 0xf4, 0xf6, 0xec, 0xfa, 0x00, 0x42, 0xdc, 0xf5, 0x57, 0xa7, 0xa9, 0xaa, 0xae, 0xfa, 0x7a, + 0x60, 0x7b, 0x59, 0x9d, 0x25, 0xf1, 0xcb, 0xfd, 0x65, 0x9e, 0x95, 0x99, 0xec, 0xc7, 0x69, 0xa9, + 0xf3, 0x34, 0x4a, 0x54, 0x01, 0x1e, 0x66, 0xb5, 0x0c, 0xa0, 0xf7, 0x24, 0x4b, 0xaa, 0x45, 0x5a, + 0x04, 0x62, 0xec, 0x85, 0x3e, 0x5a, 0x28, 0x1f, 0x40, 0xf7, 0xb3, 0xb2, 0xcc, 0x8b, 0xa0, 0x33, + 0xf6, 0xc2, 0xe1, 0xc1, 0xee, 0xbe, 0x75, 0xdd, 0x27, 0x31, 0x1a, 0xa5, 0x94, 0xe0, 0x3f, 0xd3, + 0x17, 0x45, 0xe0, 0x8d, 0xbd, 0x70, 0x80, 0x7c, 0xa6, 0x98, 0x98, 0x45, 0x79, 0x9c, 0x9e, 0x07, + 0xfe, 0x58, 0x84, 0xdb, 0x68, 0xa1, 0x7a, 0x0e, 0x83, 0x93, 0xf8, 0x3c, 0xd5, 0x73, 0xfa, 0xf4, + 0xbb, 0xe0, 0xbd, 0xc8, 0xe8, 0xb3, 0x22, 0x1c, 0x1e, 0xec, 0xb8, 0xf0, 0x98, 0xd5, 0x48, 0x1a, + 0x32, 0x98, 0xea, 0xf3, 0xa0, 0x73, 0xad, 0xc1, 0x54, 0x9f, 0xab, 0xc7, 0xb0, 0x8b, 0x59, 0x3d, + 0x99, 0xeb, 0xb4, 0x8c, 0x5f, 0xc5, 0xda, 0xa4, 0x83, 0x59, 0x6d, 0x6b, 0xe1, 0xf3, 0x2a, 0xc5, + 0x8e, 0x4b, 0x51, 0x7d, 0x0c, 0xfe, 0x8b, 0x28, 0xce, 0xe5, 0x2e, 0x74, 0x26, 0x47, 0x9c, 0x82, + 0x8f, 0x9d, 0xc9, 0x91, 0x1c, 0x41, 0xf7, 0x49, 0x56, 0xa5, 0x25, 0x7f, 0xd4, 0x47, 0x03, 0xe4, + 0x1d, 0xf0, 0x9e, 0xe9, 0x8b, 0xc0, 0x1b, 0x8b, 0x70, 0x80, 0x74, 0x54, 0xc7, 0x30, 0x20, 0xff, + 0xa7, 0xb1, 0x4e, 0xe6, 0x52, 0x99, 0x60, 0x4d, 0x25, 0xad, 0x46, 0x91, 0x14, 0xcd, 0x87, 0x46, + 0xd0, 0x65, 0x63, 0x0e, 0x3c, 0x40, 0x03, 0xd4, 0x17, 0x00, 0xa4, 0x2d, 0x4c, 0x9c, 0x07, 0xd0, + 0x65, 0xc4, 0xd9, 0x5f, 0x0d, 0x64, 0x94, 0x37, 0x44, 0x9a, 0x42, 0x9f, 0x0f, 0xd4, 0xd8, 0x95, + 0x85, 0x68, 0x59, 0x90, 0x94, 0x9a, 0x75, 0x64, 0x4b, 0x63, 0x20, 0xef, 0xc1, 0x16, 0x66, 0xb5, + 0xab, 0xae, 0x41, 0xea, 0x6b, 0x80, 0xcf, 0xf3, 0xac, 0x5a, 0x9a, 0x06, 0x84, 0xd0, 0x65, 0xd4, + 0x64, 0x26, 0x5d, 0x66, 0xf6, 0xa3, 0x68, 0x0c, 0x6e, 0x6e, 0xe0, 0x49, 0xb5, 0xe0, 0x4f, 0x78, + 0x48, 0x47, 0x75, 0x00, 0xfd, 0x59, 0x94, 0xac, 0xb4, 0xb3, 0x28, 0xe1, 0x6c, 0x3d, 0xa4, 0xe3, + 0x7a, 0x14, 0xaf, 0x89, 0xa2, 0xbe, 0x82, 0x1d, 0x33, 0x9c, 0x34, 0x7a, 0x27, 0xba, 0xbc, 0x72, + 0x7b, 0xff, 0x6c, 0x64, 0xaf, 0xde, 0xe6, 0x8f, 0x02, 0x7c, 0xd2, 0x59, 0x95, 0x58, 0xa9, 0x68, + 0x78, 0x4e, 0x2f, 0x96, 0xba, 0x29, 0x87, 0xcf, 0x72, 0x0c, 0xc3, 0x93, 0x92, 0xe6, 0x79, 0x16, + 0x25, 0x95, 0x6e, 0x02, 0xb5, 0x45, 0xf2, 0x1d, 0xe8, 0x4f, 0xd2, 0xd2, 0xa8, 0x7d, 0x2e, 0x61, + 0x85, 0xe5, 0x7d, 0x18, 0x1c, 0x66, 0x59, 0x62, 0x94, 0xdd, 0xb1, 0x08, 0xfb, 0xe8, 0x04, 0x72, + 0x0f, 0xe0, 0x69, 0x92, 0x45, 0x8d, 0xef, 0xd6, 0x58, 0x84, 0x02, 0x5b, 0x12, 0xf5, 0x10, 0x7a, + 0x94, 0xe9, 0xf3, 0x68, 0xe9, 0xaa, 0x15, 0xb7, 0x54, 0xab, 0xfe, 0x14, 0xb0, 0xfd, 0x65, 0xa5, + 0xf3, 0x0b, 0xd4, 0xdf, 0x54, 0xba, 0x28, 0xa9, 0xb7, 0x8c, 0xed, 0x74, 0x30, 0xa0, 0x39, 0x38, + 0x79, 0x1d, 0xe5, 0x73, 0xd3, 0x3b, 0x1f, 0x1b, 0x44, 0xb5, 0xba, 0x9e, 0x17, 0x5c, 0x6b, 0x1f, + 0xdb, 0x22, 0x9e, 0x20, 0xbd, 0xc8, 0x4a, 0x5b, 0x4c, 0x83, 0x64, 0x08, 0xff, 0x3f, 0x7e, 0xf3, + 0x32, 0xa9, 0xe6, 0x1a, 0xb3, 0xda, 0x78, 0x6f, 0xb1, 0xc1, 0xa6, 0x58, 0xbe, 0x07, 0xbb, 0x8d, + 0xc8, 0x52, 0x51, 0x8f, 0x0d, 0x37, 0xa4, 0xf2, 0x11, 0x6c, 0x1f, 0x2f, 0xce, 0xf4, 0x7c, 0xae, + 0xe7, 0x47, 0x51, 0x19, 0x05, 0x7d, 0xae, 0x7b, 0x83, 0x18, 0xd6, 0x4c, 0xd4, 0x77, 0x02, 0x76, + 0x9a, 0xea, 0x8b, 0x65, 0x96, 0x16, 0x9a, 0xae, 0xf8, 0x38, 0xcf, 0xed, 0x15, 0x1f, 0xe7, 0xb9, + 0x7c, 0x08, 0x3d, 0xd4, 0x45, 0x95, 0x94, 0x76, 0x6e, 0xee, 0xba, 0x88, 0xd6, 0xb7, 0x4a, 0x4a, + 0xb4, 0x56, 0xf2, 0x13, 0xd8, 0x5d, 0x9b, 0x43, 0xc3, 0x7e, 0xc3, 0x83, 0xb7, 0x9d, 0xdf, 0x9a, + 0x1e, 0x37, 0xcc, 0xd5, 0x4f, 0x1e, 0x0c, 0x5b, 0x91, 0x89, 0xe8, 0x30, 0xab, 0x6f, 0x60, 0x42, + 0xda, 0xe8, 0x6d, 0x10, 0xd3, 0x66, 0x04, 0xc5, 0xd4, 0xf1, 0x84, 0x77, 0x1b, 0x4f, 0x10, 0xb3, + 0xbf, 0x8e, 0xd2, 0x73, 0x3d, 0xe7, 0x11, 0xec, 0xa3, 0x85, 0x72, 0xdf, 0xed, 0x1e, 0xdf, 0xd9, + 0xda, 0x42, 0x5b, 0x0d, 0xba, 0xfd, 0xb4, 0x3b, 0x40, 0xd7, 0xb7, 0xd3, 0xec, 0x80, 0xe1, 0x8d, + 0xc9, 0x11, 0xdd, 0x15, 0xcf, 0x8b, 0x41, 0xf2, 0x23, 0x18, 0x3a, 0xde, 0x28, 0x9a, 0x2b, 0x1a, + 0xb9, 0xf0, 0x4e, 0x89, 0x6d, 0x43, 0xf9, 0xe9, 0x26, 0x95, 0x07, 0x03, 0xce, 0x2c, 0x58, 0xeb, + 0x46, 0x4b, 0x8f, 0x9b, 0xd4, 0xff, 0xa8, 0xf5, 0xb6, 0x04, 0xc0, 0xce, 0x6f, 0x39, 0xe7, 0x95, + 0x0a, 0x5b, 0x2f, 0xd0, 0x87, 0x6d, 0xfa, 0x0d, 0x86, 0xec, 0x33, 0x5a, 0xef, 0xa6, 0xd1, 0x61, + 0xcb, 0x4e, 0xfd, 0x2e, 0x60, 0x67, 0xb2, 0x58, 0x66, 0x79, 0xd9, 0x5a, 0xa9, 0x49, 0x3a, 0xd7, + 0x6f, 0xec, 0x4a, 0x31, 0xb8, 0x9e, 0xa8, 0x49, 0xca, 0xab, 0xc5, 0xab, 0xe4, 0xa3, 0x01, 0xad, + 0x76, 0xfa, 0x6b, 0xed, 0xbc, 0x0f, 0x03, 0x33, 0x3b, 0xa4, 0xea, 0xb2, 0xca, 0x09, 0x88, 0x2c, + 0x4e, 0xe3, 0x85, 0x2e, 0xca, 0x68, 0xb1, 0xa4, 0xed, 0xf2, 0x42, 0x0f, 0x5b, 0x12, 0xf3, 0x10, + 0xd7, 0xfc, 0xf8, 0xf5, 0xf8, 0xf1, 0xb3, 0x90, 0x3c, 0x4d, 0x18, 0x56, 0xf6, 0x59, 0xd9, 0x92, + 0xa8, 0x5f, 0x04, 0x48, 0x53, 0x23, 0xd3, 0xce, 0x7f, 0x57, 0xe8, 0xed, 0x05, 0xdd, 0x83, 0x2d, + 0xfe, 0x9e, 0x2d, 0xa6, 0x41, 0x1b, 0xe9, 0xf6, 0x36, 0xd3, 0x25, 0x96, 0x72, 0x1c, 0x69, 0xea, + 0x11, 0xd8, 0x16, 0xa9, 0x19, 0x8c, 0x4e, 0xf3, 0x28, 0x2d, 0x92, 0xa8, 0xd4, 0xe4, 0xf2, 0x6f, + 0x2a, 0xba, 0xe6, 0x5f, 0x47, 0xbd, 0x0f, 0x77, 0x37, 0xe2, 0x3a, 0x9e, 0xa1, 0x12, 0x3d, 0x2e, + 0x91, 0x8e, 0xea, 0x10, 0x82, 0x66, 0x6c, 0xcc, 0xdf, 0x50, 0x93, 0xc2, 0x2c, 0xd6, 0x35, 0x85, + 0x9e, 0x46, 0x0b, 0xdd, 0x64, 0xc1, 0x67, 0x92, 0x31, 0xcd, 0x75, 0xf8, 0x1f, 0x8a, 0xcf, 0xea, + 0x15, 0x8c, 0xae, 0x8b, 0xc1, 0x0f, 0x66, 0xa2, 0x23, 0xc3, 0x6b, 0x7d, 0x34, 0x40, 0x3e, 0x86, + 0xee, 0xb7, 0xb1, 0xae, 0x2d, 0xaf, 0x29, 0x37, 0xda, 0x37, 0x25, 0x82, 0xc6, 0x41, 0xfd, 0x20, + 0x6c, 0xb2, 0x2d, 0xaa, 0xff, 0xdb, 0x9e, 0x99, 0xfb, 0x6e, 0xde, 0x6c, 0x73, 0xdf, 0x81, 0x79, + 0xaf, 0xdc, 0x83, 0x6b, 0x21, 0xbd, 0x91, 0x74, 0x9c, 0x45, 0x89, 0x19, 0xfa, 0x01, 0xae, 0xf0, + 0xed, 0x53, 0x72, 0x78, 0xe7, 0xe7, 0xcb, 0x3d, 0xf1, 0xeb, 0xe5, 0x9e, 0xf8, 0xed, 0x72, 0x4f, + 0x7c, 0xff, 0xc7, 0xde, 0xff, 0xce, 0xb6, 0xf8, 0xef, 0xf6, 0x83, 0xbf, 0x02, 0x00, 0x00, 0xff, + 0xff, 0xdd, 0x9e, 0xfe, 0xf7, 0xed, 0x0a, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index b8d569f85..a58c75e96 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -25,6 +25,16 @@ message Pair { uint64 Count = 2; } +message PairField { + Pair Pair = 1; + string Field = 2; +} + +message PairsField { + repeated Pair Pairs = 1; + string Field = 2; +} + message FieldRow{ string Field = 1; uint64 RowID = 2; @@ -88,6 +98,7 @@ message QueryResult { repeated GroupCount GroupCounts = 8; RowIdentifiers RowIdentifiers = 9; SignedRow SignedRow = 10; + PairsField PairsField = 11; } message ImportRequest { @@ -137,4 +148,4 @@ message ImportColumnAttrsRequest { string AttrKey = 3; repeated string AttrVals = 4; repeated uint64 ColumnIDs = 5; -} \ No newline at end of file +} diff --git a/pql/ast.go b/pql/ast.go index 7749c02dd..738601100 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pilosa/pilosa/v2/ext" + "github.com/molecula/ext" ) // Query represents a PQL query. @@ -347,10 +347,17 @@ var callInfoByFunc = map[string]callInfo{ "Difference": {allowUnknown: false}, "Intersect": {allowUnknown: false}, "Not": {allowUnknown: false}, - "ClearRow": {allowUnknown: true}, - "Store": {allowUnknown: true}, - "MinRow": allowField, - "MaxRow": allowField, + "All": { + allowUnknown: false, + prototypes: map[string]interface{}{ + "limit": int64(0), + "offset": int64(0), + }, + }, + "ClearRow": {allowUnknown: true}, + "Store": {allowUnknown: true}, + "MinRow": allowField, + "MaxRow": allowField, "Rows": { allowUnknown: false, prototypes: map[string]interface{}{ diff --git a/proto/interface.go b/proto/interface.go index 0a3913921..61a5dcd49 100644 --- a/proto/interface.go +++ b/proto/interface.go @@ -14,10 +14,254 @@ package pilosa +import ( + "errors" + "fmt" + "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// StreamClient is an interface for a stream +// which can return a RowResponse sent to a +// stream via Send(). type StreamClient interface { Recv() (*RowResponse, error) } +// StreamServer is an interface for a stream +// which can accept a RowResponse to be later +// returned by the stream via Recv(). type StreamServer interface { Send(*RowResponse) error } + +// EOF acts as an io.EOF encoded into a RowResponse. +var EOF *RowResponse = &RowResponse{ + StatusError: &StatusError{ + Code: 0, + Message: "EOF", + }, +} + +// Error is a helper function to create a RowResponse +// based on an error message. If the error is a grpc +// Status, then the status code is passed through. +func Error(err error) *RowResponse { + status, _ := status.FromError(err) + return &RowResponse{ + StatusError: &StatusError{ + Code: uint32(status.Code()), + Message: status.Err().Error(), + }, + } +} + +// ErrorWrap prepends a message to the existing status +// error message. +func ErrorWrap(err error, message string) *RowResponse { + status, _ := status.FromError(err) + return &RowResponse{ + StatusError: &StatusError{ + Code: uint32(status.Code()), + Message: message + ": " + status.Err().Error(), + }, + } +} + +// ErrorWrapf prepends a message to the existing status +// error message with the format specifier. +func ErrorWrapf(err error, format string, args ...interface{}) *RowResponse { + status, _ := status.FromError(err) + return &RowResponse{ + StatusError: &StatusError{ + Code: uint32(status.Code()), + Message: fmt.Sprintf(format, args...) + ": " + status.Err().Error(), + }, + } +} + +// ErrorCode is a helper function to create a RowResponse +// based on a grpc status code and an error message. +func ErrorCode(err error, c codes.Code) *RowResponse { + return &RowResponse{ + StatusError: &StatusError{ + Code: uint32(c), + Message: err.Error(), + }, + } +} + +// RowResponseSorter implements the sort interface for a +// provided []RowResponse based on the column index, type, +// and sort direction. +type RowResponseSorter struct { + colIdx []int + colDescending []bool + colType []string + + rrs []*RowResponse +} + +// NewRowResponseSorter return a new RowResponseSorter. It +// does input validation and returns an error if the inputs +// aren't compatible. +func NewRowResponseSorter(idxs []int, dirs []bool, typs []string, rrs []*RowResponse) (*RowResponseSorter, error) { + // Ensure the input slices are non-empty and equal size. + if len(idxs) == 0 { + return nil, errors.New("index list cannot be empty") + } + if len(dirs) != len(idxs) || len(typs) != len(idxs) { + return nil, errors.New("index, direction, and type lists must be the same size") + } + + // Ensure the provided data types are supported by the sorter. + for i := range typs { + switch typs[i] { + case "[]uint64", "[]string", "bool", "float64", "int64", "string", "uint64": + // pass + default: + return nil, fmt.Errorf("unsupported data type: %s", typs[i]) + } + } + + // Ensure max(colIdx) is within size of rr.Columns. + if len(rrs) > 0 { + var maxColIdx int + for i := range idxs { + if idxs[i] > maxColIdx { + maxColIdx = idxs[i] + } + } + if maxColIdx >= len(rrs[0].Columns) { + return nil, fmt.Errorf("column index is out of range: %d", maxColIdx) + } + } + + return &RowResponseSorter{ + colIdx: idxs, + colDescending: dirs, + colType: typs, + rrs: rrs, + }, nil + +} + +func (r RowResponseSorter) Len() int { return len(r.rrs) } +func (r RowResponseSorter) Swap(i, j int) { r.rrs[i], r.rrs[j] = r.rrs[j], r.rrs[i] } +func (r RowResponseSorter) Less(i, j int) bool { + ri := r.rrs[i] + rj := r.rrs[j] + + for i, idx := range r.colIdx { + coli := ri.Columns[idx] + colj := rj.Columns[idx] + var comp int + switch r.colType[i] { + case "[]uint64": + ai := coli.GetUint64ArrayVal().Vals + aj := colj.GetUint64ArrayVal().Vals + comp = func() int { + for ii := 0; ii < len(ai); ii++ { + if len(aj) == ii { + return 1 + } + piv := ai[ii] + pjv := aj[ii] + if piv == pjv { + continue + } else if piv < pjv { + return -1 + } else { + return 1 + } + } + if len(aj) > len(ai) { + return -1 + } + return 0 + }() + case "[]string": + ai := coli.GetStringArrayVal().Vals + aj := colj.GetStringArrayVal().Vals + comp = func() int { + for ii := 0; ii < len(ai); ii++ { + if len(aj) == ii { + return 1 + } + sComp := strings.Compare(ai[ii], aj[ii]) + if sComp == 0 { + continue + } else { + return sComp + } + } + if len(aj) > len(ai) { + return -1 + } + return 0 + }() + case "bool": + bi := coli.GetBoolVal() + bj := colj.GetBoolVal() + if bi == bj { + comp = 0 + } else if !bi && bj { + comp = -1 + } else { + comp = 1 + } + case "float64": + fi := coli.GetFloat64Val() + fj := colj.GetFloat64Val() + if fi == fj { + comp = 0 + } else if fi < fj { + comp = -1 + } else { + comp = 1 + } + case "int64": + ni := coli.GetInt64Val() + nj := colj.GetInt64Val() + if ni == nj { + comp = 0 + } else if ni < nj { + comp = -1 + } else { + comp = 1 + } + case "string": + comp = strings.Compare(coli.GetStringVal(), colj.GetStringVal()) + case "uint64": + ni := coli.GetUint64Val() + nj := colj.GetUint64Val() + if ni == nj { + comp = 0 + } else if ni < nj { + comp = -1 + } else { + comp = 1 + } + } + + isDescending := r.colDescending[i] + + switch comp { + case 0: + continue + case -1: + if isDescending { + return false + } + return true + case 1: + if isDescending { + return true + } + return false + } + } + return false +} diff --git a/proto/pilosa.pb.go b/proto/pilosa.pb.go index b81722e06..2cec94582 100644 --- a/proto/pilosa.pb.go +++ b/proto/pilosa.pb.go @@ -4,15 +4,16 @@ package pilosa import ( - context "context" fmt "fmt" proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" math "math" ) +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf @@ -22,7 +23,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type QueryPQLRequest struct { Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` @@ -71,9 +72,57 @@ func (m *QueryPQLRequest) GetPql() string { return "" } +type StatusError struct { + Code uint32 `protobuf:"varint,1,opt,name=Code,proto3" json:"Code,omitempty"` + Message string `protobuf:"bytes,2,opt,name=Message,proto3" json:"Message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *StatusError) Reset() { *m = StatusError{} } +func (m *StatusError) String() string { return proto.CompactTextString(m) } +func (*StatusError) ProtoMessage() {} +func (*StatusError) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{1} +} + +func (m *StatusError) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_StatusError.Unmarshal(m, b) +} +func (m *StatusError) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_StatusError.Marshal(b, m, deterministic) +} +func (m *StatusError) XXX_Merge(src proto.Message) { + xxx_messageInfo_StatusError.Merge(m, src) +} +func (m *StatusError) XXX_Size() int { + return xxx_messageInfo_StatusError.Size(m) +} +func (m *StatusError) XXX_DiscardUnknown() { + xxx_messageInfo_StatusError.DiscardUnknown(m) +} + +var xxx_messageInfo_StatusError proto.InternalMessageInfo + +func (m *StatusError) GetCode() uint32 { + if m != nil { + return m.Code + } + return 0 +} + +func (m *StatusError) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + type RowResponse struct { Headers []*ColumnInfo `protobuf:"bytes,1,rep,name=headers,proto3" json:"headers,omitempty"` Columns []*ColumnResponse `protobuf:"bytes,2,rep,name=columns,proto3" json:"columns,omitempty"` + StatusError *StatusError `protobuf:"bytes,3,opt,name=StatusError,proto3" json:"StatusError,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -83,7 +132,7 @@ func (m *RowResponse) Reset() { *m = RowResponse{} } func (m *RowResponse) String() string { return proto.CompactTextString(m) } func (*RowResponse) ProtoMessage() {} func (*RowResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{1} + return fileDescriptor_ef0691a44d1e275c, []int{2} } func (m *RowResponse) XXX_Unmarshal(b []byte) error { @@ -118,6 +167,13 @@ func (m *RowResponse) GetColumns() []*ColumnResponse { return nil } +func (m *RowResponse) GetStatusError() *StatusError { + if m != nil { + return m.StatusError + } + return nil +} + type ColumnInfo struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Datatype string `protobuf:"bytes,2,opt,name=datatype,proto3" json:"datatype,omitempty"` @@ -130,7 +186,7 @@ func (m *ColumnInfo) Reset() { *m = ColumnInfo{} } func (m *ColumnInfo) String() string { return proto.CompactTextString(m) } func (*ColumnInfo) ProtoMessage() {} func (*ColumnInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{2} + return fileDescriptor_ef0691a44d1e275c, []int{3} } func (m *ColumnInfo) XXX_Unmarshal(b []byte) error { @@ -185,7 +241,7 @@ func (m *ColumnResponse) Reset() { *m = ColumnResponse{} } func (m *ColumnResponse) String() string { return proto.CompactTextString(m) } func (*ColumnResponse) ProtoMessage() {} func (*ColumnResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{3} + return fileDescriptor_ef0691a44d1e275c, []int{4} } func (m *ColumnResponse) XXX_Unmarshal(b []byte) error { @@ -321,9 +377,9 @@ func (m *ColumnResponse) GetFloat64Val() float64 { return 0 } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*ColumnResponse) XXX_OneofWrappers() []interface{} { - return []interface{}{ +// XXX_OneofFuncs is for the internal use of the proto package. +func (*ColumnResponse) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _ColumnResponse_OneofMarshaler, _ColumnResponse_OneofUnmarshaler, _ColumnResponse_OneofSizer, []interface{}{ (*ColumnResponse_StringVal)(nil), (*ColumnResponse_Uint64Val)(nil), (*ColumnResponse_Int64Val)(nil), @@ -335,10 +391,162 @@ func (*ColumnResponse) XXX_OneofWrappers() []interface{} { } } +func _ColumnResponse_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*ColumnResponse) + // columnVal + switch x := m.ColumnVal.(type) { + case *ColumnResponse_StringVal: + b.EncodeVarint(1<<3 | proto.WireBytes) + b.EncodeStringBytes(x.StringVal) + case *ColumnResponse_Uint64Val: + b.EncodeVarint(2<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Uint64Val)) + case *ColumnResponse_Int64Val: + b.EncodeVarint(3<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Int64Val)) + case *ColumnResponse_BoolVal: + t := uint64(0) + if x.BoolVal { + t = 1 + } + b.EncodeVarint(4<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *ColumnResponse_BlobVal: + b.EncodeVarint(5<<3 | proto.WireBytes) + b.EncodeRawBytes(x.BlobVal) + case *ColumnResponse_Uint64ArrayVal: + b.EncodeVarint(6<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Uint64ArrayVal); err != nil { + return err + } + case *ColumnResponse_StringArrayVal: + b.EncodeVarint(7<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.StringArrayVal); err != nil { + return err + } + case *ColumnResponse_Float64Val: + b.EncodeVarint(8<<3 | proto.WireFixed64) + b.EncodeFixed64(math.Float64bits(x.Float64Val)) + case nil: + default: + return fmt.Errorf("ColumnResponse.ColumnVal has unexpected type %T", x) + } + return nil +} + +func _ColumnResponse_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*ColumnResponse) + switch tag { + case 1: // columnVal.stringVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.ColumnVal = &ColumnResponse_StringVal{x} + return true, err + case 2: // columnVal.uint64Val + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_Uint64Val{x} + return true, err + case 3: // columnVal.int64Val + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_Int64Val{int64(x)} + return true, err + case 4: // columnVal.boolVal + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_BoolVal{x != 0} + return true, err + case 5: // columnVal.blobVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.ColumnVal = &ColumnResponse_BlobVal{x} + return true, err + case 6: // columnVal.uint64ArrayVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Uint64Array) + err := b.DecodeMessage(msg) + m.ColumnVal = &ColumnResponse_Uint64ArrayVal{msg} + return true, err + case 7: // columnVal.stringArrayVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(StringArray) + err := b.DecodeMessage(msg) + m.ColumnVal = &ColumnResponse_StringArrayVal{msg} + return true, err + case 8: // columnVal.float64Val + if wire != proto.WireFixed64 { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeFixed64() + m.ColumnVal = &ColumnResponse_Float64Val{math.Float64frombits(x)} + return true, err + default: + return false, nil + } +} + +func _ColumnResponse_OneofSizer(msg proto.Message) (n int) { + m := msg.(*ColumnResponse) + // columnVal + switch x := m.ColumnVal.(type) { + case *ColumnResponse_StringVal: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.StringVal))) + n += len(x.StringVal) + case *ColumnResponse_Uint64Val: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Uint64Val)) + case *ColumnResponse_Int64Val: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Int64Val)) + case *ColumnResponse_BoolVal: + n += 1 // tag and wire + n += 1 + case *ColumnResponse_BlobVal: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.BlobVal))) + n += len(x.BlobVal) + case *ColumnResponse_Uint64ArrayVal: + s := proto.Size(x.Uint64ArrayVal) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *ColumnResponse_StringArrayVal: + s := proto.Size(x.StringArrayVal) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *ColumnResponse_Float64Val: + n += 1 // tag and wire + n += 8 + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + type InspectRequest struct { Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` Columns *IdsOrKeys `protobuf:"bytes,2,opt,name=columns,proto3" json:"columns,omitempty"` FilterFields []string `protobuf:"bytes,3,rep,name=filterFields,proto3" json:"filterFields,omitempty"` + Limit uint64 `protobuf:"varint,4,opt,name=limit,proto3" json:"limit,omitempty"` + Offset uint64 `protobuf:"varint,5,opt,name=offset,proto3" json:"offset,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -348,7 +556,7 @@ func (m *InspectRequest) Reset() { *m = InspectRequest{} } func (m *InspectRequest) String() string { return proto.CompactTextString(m) } func (*InspectRequest) ProtoMessage() {} func (*InspectRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{4} + return fileDescriptor_ef0691a44d1e275c, []int{5} } func (m *InspectRequest) XXX_Unmarshal(b []byte) error { @@ -390,6 +598,20 @@ func (m *InspectRequest) GetFilterFields() []string { return nil } +func (m *InspectRequest) GetLimit() uint64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *InspectRequest) GetOffset() uint64 { + if m != nil { + return m.Offset + } + return 0 +} + type Uint64Array struct { Vals []uint64 `protobuf:"varint,1,rep,packed,name=vals,proto3" json:"vals,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -401,7 +623,7 @@ func (m *Uint64Array) Reset() { *m = Uint64Array{} } func (m *Uint64Array) String() string { return proto.CompactTextString(m) } func (*Uint64Array) ProtoMessage() {} func (*Uint64Array) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{5} + return fileDescriptor_ef0691a44d1e275c, []int{6} } func (m *Uint64Array) XXX_Unmarshal(b []byte) error { @@ -440,7 +662,7 @@ func (m *StringArray) Reset() { *m = StringArray{} } func (m *StringArray) String() string { return proto.CompactTextString(m) } func (*StringArray) ProtoMessage() {} func (*StringArray) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{6} + return fileDescriptor_ef0691a44d1e275c, []int{7} } func (m *StringArray) XXX_Unmarshal(b []byte) error { @@ -482,7 +704,7 @@ func (m *IdsOrKeys) Reset() { *m = IdsOrKeys{} } func (m *IdsOrKeys) String() string { return proto.CompactTextString(m) } func (*IdsOrKeys) ProtoMessage() {} func (*IdsOrKeys) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{7} + return fileDescriptor_ef0691a44d1e275c, []int{8} } func (m *IdsOrKeys) XXX_Unmarshal(b []byte) error { @@ -540,16 +762,83 @@ func (m *IdsOrKeys) GetKeys() *StringArray { return nil } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*IdsOrKeys) XXX_OneofWrappers() []interface{} { - return []interface{}{ +// XXX_OneofFuncs is for the internal use of the proto package. +func (*IdsOrKeys) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _IdsOrKeys_OneofMarshaler, _IdsOrKeys_OneofUnmarshaler, _IdsOrKeys_OneofSizer, []interface{}{ (*IdsOrKeys_Ids)(nil), (*IdsOrKeys_Keys)(nil), } } +func _IdsOrKeys_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*IdsOrKeys) + // type + switch x := m.Type.(type) { + case *IdsOrKeys_Ids: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Ids); err != nil { + return err + } + case *IdsOrKeys_Keys: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Keys); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("IdsOrKeys.Type has unexpected type %T", x) + } + return nil +} + +func _IdsOrKeys_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*IdsOrKeys) + switch tag { + case 1: // type.ids + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Uint64Array) + err := b.DecodeMessage(msg) + m.Type = &IdsOrKeys_Ids{msg} + return true, err + case 2: // type.keys + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(StringArray) + err := b.DecodeMessage(msg) + m.Type = &IdsOrKeys_Keys{msg} + return true, err + default: + return false, nil + } +} + +func _IdsOrKeys_OneofSizer(msg proto.Message) (n int) { + m := msg.(*IdsOrKeys) + // type + switch x := m.Type.(type) { + case *IdsOrKeys_Ids: + s := proto.Size(x.Ids) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *IdsOrKeys_Keys: + s := proto.Size(x.Keys) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + func init() { proto.RegisterType((*QueryPQLRequest)(nil), "pilosa.QueryPQLRequest") + proto.RegisterType((*StatusError)(nil), "pilosa.StatusError") proto.RegisterType((*RowResponse)(nil), "pilosa.RowResponse") proto.RegisterType((*ColumnInfo)(nil), "pilosa.ColumnInfo") proto.RegisterType((*ColumnResponse)(nil), "pilosa.ColumnResponse") @@ -559,44 +848,6 @@ func init() { proto.RegisterType((*IdsOrKeys)(nil), "pilosa.IdsOrKeys") } -func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } - -var fileDescriptor_ef0691a44d1e275c = []byte{ - // 497 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x53, 0x5d, 0x6b, 0xd4, 0x40, - 0x14, 0xcd, 0x34, 0xe9, 0xee, 0xe6, 0x66, 0x59, 0xf5, 0x2a, 0x1a, 0x16, 0x91, 0x98, 0x17, 0x23, - 0x4a, 0x29, 0xab, 0x08, 0x4a, 0x7d, 0xb0, 0x82, 0x64, 0x51, 0xb0, 0x1d, 0xb1, 0xef, 0xb3, 0xcd, - 0x6c, 0x0d, 0xce, 0x66, 0xd2, 0x4c, 0xd6, 0x9a, 0x57, 0xff, 0xa2, 0x7f, 0x48, 0x66, 0xf2, 0xb1, - 0x49, 0x61, 0x7d, 0x9b, 0x39, 0xe7, 0xdc, 0xef, 0x7b, 0x61, 0x9a, 0xa7, 0x42, 0x2a, 0x76, 0x94, - 0x17, 0xb2, 0x94, 0x38, 0xaa, 0x7f, 0xe1, 0x5b, 0xb8, 0x73, 0xbe, 0xe5, 0x45, 0x75, 0x76, 0xfe, - 0x85, 0xf2, 0xeb, 0x2d, 0x57, 0x25, 0x3e, 0x80, 0xc3, 0x34, 0x4b, 0xf8, 0x6f, 0x9f, 0x04, 0x24, - 0x72, 0x69, 0xfd, 0xc1, 0xbb, 0x60, 0xe7, 0xd7, 0xc2, 0x3f, 0x30, 0x98, 0x7e, 0x86, 0x1b, 0xf0, - 0xa8, 0xbc, 0xa1, 0x5c, 0xe5, 0x32, 0x53, 0x1c, 0x5f, 0xc2, 0xf8, 0x07, 0x67, 0x09, 0x2f, 0x94, - 0x4f, 0x02, 0x3b, 0xf2, 0x16, 0x78, 0xd4, 0x44, 0xfc, 0x28, 0xc5, 0x76, 0x93, 0x2d, 0xb3, 0xb5, - 0xa4, 0xad, 0x04, 0x8f, 0x61, 0x7c, 0x69, 0x60, 0xe5, 0x1f, 0x18, 0xf5, 0xc3, 0xa1, 0xba, 0x75, - 0x4b, 0x5b, 0x59, 0x78, 0x02, 0xb0, 0x73, 0x84, 0x08, 0x4e, 0xc6, 0x36, 0xbc, 0xc9, 0xd1, 0xbc, - 0x71, 0x0e, 0x93, 0x84, 0x95, 0xac, 0xac, 0x72, 0xde, 0xe4, 0xd9, 0xfd, 0xc3, 0xbf, 0x07, 0x30, - 0x1b, 0x7a, 0xc6, 0x27, 0xe0, 0xaa, 0xb2, 0x48, 0xb3, 0xab, 0x0b, 0x26, 0x6a, 0x3f, 0xb1, 0x45, - 0x77, 0x90, 0xe6, 0xb7, 0x69, 0x56, 0xbe, 0x79, 0xad, 0x79, 0xed, 0xcf, 0xd1, 0x7c, 0x07, 0xe1, - 0x63, 0x98, 0x74, 0xb4, 0x1d, 0x90, 0xc8, 0x8e, 0x2d, 0xda, 0x21, 0x38, 0x87, 0xf1, 0x4a, 0x4a, - 0xa1, 0x49, 0x27, 0x20, 0xd1, 0x24, 0xb6, 0x68, 0x0b, 0x18, 0x4e, 0xc8, 0x95, 0xe6, 0x0e, 0x03, - 0x12, 0x4d, 0x0d, 0x57, 0x03, 0xf8, 0x1e, 0x66, 0x75, 0x88, 0x0f, 0x45, 0xc1, 0x2a, 0x2d, 0x19, - 0x05, 0x24, 0xf2, 0x16, 0xf7, 0xdb, 0xfe, 0x7c, 0xdf, 0xb1, 0xb1, 0x45, 0x6f, 0x89, 0xb5, 0x79, - 0x5d, 0x41, 0x67, 0x3e, 0x1e, 0x9a, 0x7f, 0xdb, 0xb1, 0xda, 0x7c, 0x28, 0xc6, 0x00, 0x60, 0x2d, - 0x24, 0x6b, 0xaa, 0x9a, 0x04, 0x24, 0x22, 0xb1, 0x45, 0x7b, 0xd8, 0xa9, 0x07, 0x6e, 0x3d, 0x91, - 0x0b, 0x26, 0xc2, 0x1b, 0x98, 0x2d, 0x33, 0x95, 0xf3, 0xcb, 0xf2, 0xff, 0xcb, 0xf3, 0xa2, 0x3f, - 0x6d, 0x9d, 0xce, 0xbd, 0x36, 0x9d, 0x65, 0xa2, 0xbe, 0x16, 0x9f, 0x79, 0xa5, 0xba, 0x41, 0x63, - 0x08, 0xd3, 0x75, 0x2a, 0x4a, 0x5e, 0x7c, 0x4a, 0xb9, 0x48, 0x94, 0x6f, 0x07, 0x76, 0xe4, 0xd2, - 0x01, 0x16, 0x3e, 0x05, 0xaf, 0xd7, 0x07, 0xbd, 0x0d, 0xbf, 0x98, 0xa8, 0x17, 0xcf, 0xa1, 0xe6, - 0xad, 0x25, 0xbd, 0x5a, 0x07, 0x12, 0xb7, 0x91, 0x5c, 0x81, 0xdb, 0xc5, 0xc7, 0x67, 0x60, 0xa7, - 0x89, 0x32, 0x79, 0xef, 0xed, 0xb6, 0x56, 0xe0, 0x73, 0x70, 0x7e, 0xf2, 0xaa, 0xad, 0x64, 0x4f, - 0x63, 0x8d, 0xe4, 0x74, 0x04, 0x8e, 0xde, 0xbe, 0xc5, 0x1f, 0x02, 0xa3, 0x33, 0x23, 0xc3, 0x13, - 0x98, 0xb4, 0x07, 0x87, 0x8f, 0x5a, 0xdb, 0x5b, 0x27, 0x38, 0xef, 0x9c, 0xf6, 0x0e, 0x2c, 0xb4, - 0x8e, 0x09, 0xbe, 0x83, 0x71, 0xd3, 0x70, 0xec, 0x0e, 0x66, 0x38, 0x81, 0xbd, 0xb6, 0xab, 0x91, - 0xb9, 0xfc, 0x57, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0xca, 0x68, 0x70, 0x08, 0x09, 0x04, 0x00, - 0x00, -} - // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn @@ -691,17 +942,6 @@ type PilosaServer interface { Inspect(*InspectRequest, Pilosa_InspectServer) error } -// UnimplementedPilosaServer can be embedded to have forward compatible implementations. -type UnimplementedPilosaServer struct { -} - -func (*UnimplementedPilosaServer) QueryPQL(req *QueryPQLRequest, srv Pilosa_QueryPQLServer) error { - return status.Errorf(codes.Unimplemented, "method QueryPQL not implemented") -} -func (*UnimplementedPilosaServer) Inspect(req *InspectRequest, srv Pilosa_InspectServer) error { - return status.Errorf(codes.Unimplemented, "method Inspect not implemented") -} - func RegisterPilosaServer(s *grpc.Server, srv PilosaServer) { s.RegisterService(&_Pilosa_serviceDesc, srv) } @@ -766,3 +1006,45 @@ var _Pilosa_serviceDesc = grpc.ServiceDesc{ }, Metadata: "pilosa.proto", } + +func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } + +var fileDescriptor_ef0691a44d1e275c = []byte{ + // 568 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x54, 0xdd, 0x6e, 0xd3, 0x30, + 0x14, 0x8e, 0x97, 0x2c, 0x69, 0x4e, 0xc6, 0x00, 0x83, 0x46, 0x54, 0x21, 0x14, 0x72, 0x43, 0x10, + 0x68, 0x9a, 0xca, 0x8f, 0x04, 0x8c, 0x0b, 0x36, 0x81, 0x5a, 0x01, 0x62, 0x33, 0x62, 0xf7, 0xee, + 0xe2, 0x96, 0x08, 0x37, 0xce, 0x62, 0x17, 0xe8, 0x2d, 0xcf, 0x02, 0x4f, 0xc4, 0x0b, 0x21, 0x3b, + 0x3f, 0x4d, 0x2a, 0x95, 0x3b, 0x9f, 0xef, 0xfb, 0xce, 0xf1, 0xf9, 0xb3, 0x61, 0xaf, 0xc8, 0xb8, + 0x90, 0xf4, 0xb0, 0x28, 0x85, 0x12, 0xd8, 0xad, 0xac, 0xf8, 0x05, 0x5c, 0x3f, 0x5f, 0xb2, 0x72, + 0x75, 0x76, 0xfe, 0x81, 0xb0, 0xab, 0x25, 0x93, 0x0a, 0xdf, 0x86, 0xdd, 0x2c, 0x4f, 0xd9, 0xcf, + 0x10, 0x45, 0x28, 0xf1, 0x49, 0x65, 0xe0, 0x1b, 0x60, 0x17, 0x57, 0x3c, 0xdc, 0x31, 0x98, 0x3e, + 0xc6, 0xaf, 0x20, 0xf8, 0xac, 0xa8, 0x5a, 0xca, 0xb7, 0x65, 0x29, 0x4a, 0x8c, 0xc1, 0x39, 0x15, + 0x29, 0x33, 0x5e, 0xd7, 0x88, 0x39, 0xe3, 0x10, 0xbc, 0x8f, 0x4c, 0x4a, 0x3a, 0x67, 0xb5, 0x63, + 0x63, 0xc6, 0xbf, 0x11, 0x04, 0x44, 0xfc, 0x20, 0x4c, 0x16, 0x22, 0x97, 0x0c, 0x3f, 0x06, 0xef, + 0x2b, 0xa3, 0x29, 0x2b, 0x65, 0x88, 0x22, 0x3b, 0x09, 0x46, 0xf8, 0xb0, 0xce, 0xf7, 0x54, 0xf0, + 0xe5, 0x22, 0x9f, 0xe4, 0x33, 0x41, 0x1a, 0x09, 0x3e, 0x02, 0xef, 0xd2, 0xc0, 0x32, 0xdc, 0x31, + 0xea, 0x83, 0xbe, 0xba, 0x09, 0x4b, 0x1a, 0x19, 0x7e, 0xd6, 0x4b, 0x36, 0xb4, 0x23, 0x94, 0x04, + 0xa3, 0x5b, 0x8d, 0x57, 0x87, 0x22, 0x5d, 0x5d, 0x7c, 0x0c, 0xb0, 0xbe, 0x5f, 0x97, 0x98, 0xd3, + 0x05, 0xab, 0x1b, 0x63, 0xce, 0x78, 0x08, 0x83, 0x94, 0x2a, 0xaa, 0x56, 0x45, 0x53, 0x63, 0x6b, + 0xc7, 0x7f, 0x77, 0x60, 0xbf, 0x9f, 0x10, 0xbe, 0x07, 0xbe, 0x54, 0x65, 0x96, 0xcf, 0x2f, 0x28, + 0xaf, 0xe2, 0x8c, 0x2d, 0xb2, 0x86, 0x34, 0xbf, 0xcc, 0x72, 0xf5, 0xfc, 0xa9, 0xe6, 0x75, 0x3c, + 0x47, 0xf3, 0x2d, 0x84, 0xef, 0xc2, 0xa0, 0xa5, 0x75, 0x11, 0xf6, 0xd8, 0x22, 0x2d, 0x82, 0x87, + 0xe0, 0x4d, 0x85, 0xe0, 0x9a, 0x74, 0x22, 0x94, 0x0c, 0xc6, 0x16, 0x69, 0x00, 0xc3, 0x71, 0x31, + 0xd5, 0xdc, 0x6e, 0x84, 0x92, 0x3d, 0xc3, 0x55, 0x00, 0x7e, 0x0d, 0xfb, 0xd5, 0x15, 0x6f, 0xca, + 0x92, 0xae, 0xb4, 0xc4, 0xed, 0x37, 0xe8, 0xcb, 0x9a, 0x1d, 0x5b, 0x64, 0x43, 0xac, 0xdd, 0xab, + 0x0a, 0x5a, 0x77, 0x6f, 0xb3, 0xbf, 0x2d, 0xab, 0xdd, 0xfb, 0x62, 0x1c, 0x01, 0xcc, 0xb8, 0xa0, + 0x75, 0x55, 0x83, 0x08, 0x25, 0x68, 0x6c, 0x91, 0x0e, 0x76, 0x12, 0x80, 0x5f, 0x0d, 0xf2, 0x82, + 0xf2, 0xf8, 0x0f, 0x82, 0xfd, 0x49, 0x2e, 0x0b, 0x76, 0xa9, 0xfe, 0xbf, 0xb2, 0x8f, 0xba, 0x5b, + 0xa2, 0xf3, 0xb9, 0xd9, 0xe4, 0x33, 0x49, 0xe5, 0xa7, 0xf2, 0x3d, 0x5b, 0xc9, 0xf5, 0x82, 0xc4, + 0xb0, 0x37, 0xcb, 0xb8, 0x62, 0xe5, 0xbb, 0x8c, 0xf1, 0x54, 0x86, 0x76, 0x64, 0x27, 0x3e, 0xe9, + 0x61, 0xfa, 0x1a, 0x9e, 0x2d, 0x32, 0x65, 0x9a, 0xeb, 0x90, 0xca, 0xc0, 0x07, 0xe0, 0x8a, 0xd9, + 0x4c, 0x32, 0x65, 0xfa, 0xea, 0x90, 0xda, 0x8a, 0xef, 0x43, 0xd0, 0x69, 0x9b, 0x5e, 0x9e, 0xef, + 0x94, 0x57, 0xeb, 0xed, 0x10, 0x73, 0xd6, 0x92, 0x4e, 0x6b, 0x7a, 0x12, 0xbf, 0x96, 0xcc, 0xc1, + 0x6f, 0xb3, 0xc5, 0x0f, 0xc0, 0xce, 0x52, 0x69, 0xaa, 0xdc, 0x3a, 0x1c, 0xad, 0xc0, 0x0f, 0xc1, + 0xf9, 0xc6, 0x56, 0x4d, 0xdd, 0x5b, 0xe6, 0x60, 0x24, 0x27, 0x2e, 0x38, 0x7a, 0x59, 0x47, 0xbf, + 0x10, 0xb8, 0x67, 0x46, 0x86, 0x8f, 0x61, 0xd0, 0x7c, 0x0a, 0xf8, 0x4e, 0xe3, 0xbb, 0xf1, 0x4d, + 0x0c, 0xdb, 0xa0, 0x9d, 0x67, 0x1c, 0x5b, 0x47, 0x08, 0xbf, 0x04, 0xaf, 0x1e, 0x0f, 0x6e, 0x9f, + 0x65, 0x7f, 0x5e, 0x5b, 0x7d, 0xa7, 0xae, 0xf9, 0x9d, 0x9e, 0xfc, 0x0b, 0x00, 0x00, 0xff, 0xff, + 0x9e, 0xd3, 0x7d, 0xab, 0xad, 0x04, 0x00, 0x00, +} diff --git a/proto/pilosa.proto b/proto/pilosa.proto index a8822ec46..b7b30d521 100644 --- a/proto/pilosa.proto +++ b/proto/pilosa.proto @@ -6,10 +6,15 @@ message QueryPQLRequest { string pql = 2; } +message StatusError{ + uint32 Code = 1; + string Message = 2; +} message RowResponse{ repeated ColumnInfo headers = 1; repeated ColumnResponse columns = 2; + StatusError StatusError = 3; } message ColumnInfo { @@ -34,6 +39,8 @@ message InspectRequest { string index = 1; IdsOrKeys columns = 2; repeated string filterFields = 3; + uint64 limit = 4; + uint64 offset = 5; } message Uint64Array { diff --git a/roaring/roaring.go b/roaring/roaring.go index 884889a90..bb0479f8f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3688,41 +3688,43 @@ func union(a, b *Container) *Container { func unionArrayArray(a, b *Container) *Container { statsHit("union/ArrayArray") - aa, ab := a.array(), b.array() - na, nb := len(aa), len(ab) - output := make([]uint16, na+nb) - n := 0 - for i, j := 0, 0; ; { - if i >= na && j >= nb { - break - } else if i < na && j >= nb { - output[n] = aa[i] - n++ - i++ - continue - } else if i >= na && j < nb { - output[n] = ab[j] - n++ - j++ - continue - } - - va, vb := aa[i], ab[j] + if a.N() == 0 { + return b + } + if b.N() == 0 { + return a + } + s1, s2 := a.array(), b.array() + n1, n2 := len(s1), len(s2) + output := make([]uint16, 0, n1+n2) + i, j := 0, 0 + for { + va, vb := s1[i], s2[j] if va < vb { - output[n] = va - n++ + output = append(output, va) i++ } else if va > vb { - output[n] = vb - n++ + output = append(output, vb) j++ } else { - output[n] = va - n++ - i, j = i+1, j+1 + output = append(output, va) + i++ + j++ + } + // It's possible we hit the ends at the same time, + // in which case the append will copy 0 items. This + // is cheaper than performing a separate conditional + // check every time... + if j >= n2 { + output = append(output, s1[i:]...) + break + } + if i >= n1 { + output = append(output, s2[j:]...) + break } } - return NewContainerArray(output[:n]) + return NewContainerArray(output) } // unionArrayArrayInPlace does what it sounds like -- tries to combine @@ -3730,47 +3732,56 @@ func unionArrayArray(a, b *Container) *Container { // of a good array size, so it could be up to twice that size, temporarily. func unionArrayArrayInPlace(a, b *Container) *Container { statsHit("union/ArrayArrayInPlace") - aa, ab := a.array(), b.array() - na, nb := len(aa), len(ab) - output := make([]uint16, na+nb) - outN := 0 - for i, j := 0, 0; ; { - if i >= na && j >= nb { - break - } else if i < na && j >= nb { - copy(output[outN:], aa[i:]) - outN += na - i - break - } else if i >= na && j < nb { - copy(output[outN:], ab[j:]) - outN += nb - j - break + if a.N() == 0 { + if b.N() != 0 { + // for InPlace, we actually want to ensure that + // we update a, as long as it's not frozen. + a = a.Thaw() + a.setArray(b.array()) + return a.optimize() } - - va, vb := aa[i], ab[j] + return a + } + if b.N() == 0 { + return a + } + s1, s2 := a.array(), b.array() + n1, n2 := len(s1), len(s2) + output := make([]uint16, 0, n1+n2) + i, j := 0, 0 + for { + va, vb := s1[i], s2[j] if va < vb { - output[outN] = va - outN++ + output = append(output, va) i++ } else if va > vb { - output[outN] = vb - outN++ + output = append(output, vb) j++ } else { - output[outN] = va - outN++ + output = append(output, va) i++ j++ } + // It's possible we hit the ends at the same time, + // in which case the append will copy 0 items. This + // is cheaper than performing a separate conditional + // check every time... + if j >= n2 { + output = append(output, s1[i:]...) + break + } + if i >= n1 { + output = append(output, s2[j:]...) + break + } } // a union can't omit anything that was previously in a, so if // the output is the same length, nothing changed. if len(output) != int(a.N()) { a = a.Thaw() - a.setArray(output[:outN]) - a = a.optimize() + a.setArray(output) } - return a + return a.optimize() } // unionArrayRun optimistically assumes that the result will be a run container, diff --git a/row.go b/row.go index d9a4f9cc9..82a67cf8f 100644 --- a/row.go +++ b/row.go @@ -18,7 +18,7 @@ import ( "encoding/json" "sort" - "github.com/pilosa/pilosa/v2/ext" + "github.com/molecula/ext" "github.com/pilosa/pilosa/v2/roaring" "github.com/pkg/errors" ) diff --git a/server.go b/server.go index ddf43ed50..96a4d5cde 100644 --- a/server.go +++ b/server.go @@ -17,19 +17,19 @@ package pilosa import ( "context" "fmt" - "io" "log" "os" "os/exec" "path/filepath" - "plugin" "runtime" "strconv" "strings" "sync" "time" - "github.com/pilosa/pilosa/v2/ext" + "github.com/molecula/ext" + // extensions pulls in some extensions depending on build tags + _ "github.com/pilosa/pilosa/v2/extensions" "github.com/pilosa/pilosa/v2/logger" "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" @@ -61,7 +61,6 @@ type Server struct { // nolint: maligned hosts []string clusterDisabled bool serializer Serializer - extensionPath string extensions []*ext.ExtensionInfo // External @@ -341,8 +340,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { if err != nil { return nil, err } - s.extensionPath = filepath.Join(path, ".extensions") - s.holder.Path = path // s.holder.translateFile.Path = filepath.Join(path, ".keys") s.holder.Logger = s.logger @@ -383,7 +380,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.cluster.broadcaster = s s.cluster.maxWritesPerRequest = s.maxWritesPerRequest s.holder.broadcaster = s - err = s.loadPlugins() + err = s.loadExtensions() if err != nil { s.logger.Printf("not all plugins loaded successfully") } @@ -420,67 +417,20 @@ func (s *Server) InternalClient() InternalClient { return s.defaultClient } -func (s *Server) loadPlugins() error { - var anyError error - dir, err := os.Open(s.extensionPath) - if err != nil { - // don't complain about it not existing, that's fine. - if os.IsNotExist(err) { - s.logger.Printf("extension interface v0: no extensions directory.") - return nil - } - return errors.Wrap(err, "opening extension path:") - } - defer dir.Close() - for files, err := dir.Readdir(64); err != io.EOF; files, err = dir.Readdir(64) { - if err != nil { - return errors.Wrap(err, "searching extension directory:") - } - for _, file := range files { - name := file.Name() - // only .so files are likely plugins. - if !strings.HasSuffix(name, ".so") { - continue - } - // only regular files are candidates for loading. - mode := file.Mode() - if !mode.IsRegular() { - s.logger.Printf("extension file '%s' is not a regular file", name) - continue - } - err = s.loadPlugin(name) - if err != nil { - s.logger.Printf("loading extension %s: %v", name, err) - anyError = err - } +func (s *Server) loadExtensions() error { + exts := ext.NewExtensions() + var lastError error + for _, extension := range exts { + if err := s.loadExtension(extension); err != nil { + lastError = err } } - return anyError + return lastError } -func (s *Server) loadPlugin(name string) error { - path := filepath.Join(s.extensionPath, name) - p, err := plugin.Open(path) - if err != nil { - return err - } - pluginExtInfo, err := p.Lookup("ExtensionInfo") - if err != nil { - return fmt.Errorf("%s: no ExtensionInfo found", name) - } - extInfoFunc, ok := pluginExtInfo.(func(string) (*ext.ExtensionInfo, error)) - if !ok { - return fmt.Errorf("%s: unexpected %T instead of ExtensionInfo object", name, pluginExtInfo) - } - extInfo, err := extInfoFunc("v0") - if err != nil { - return errors.Wrap(err, name) - } - if extInfo == nil { - return fmt.Errorf("%s: nil ExtensionInfo", name) - } +func (s *Server) loadExtension(extInfo *ext.ExtensionInfo) error { if extInfo.ExtensionAPI != "v0" { - return fmt.Errorf("%s: unsupported extension API %s", name, extInfo.ExtensionAPI) + return fmt.Errorf("%s: unsupported extension API %s", extInfo.Name, extInfo.ExtensionAPI) } s.extensions = append(s.extensions, extInfo) bitmapOps := extInfo.BitmapOps @@ -500,7 +450,7 @@ func (s *Server) loadPlugin(name string) error { unknownOps++ } } - err = s.executor.registerOps(bitmapOps) + err := s.executor.registerOps(bitmapOps) if err != nil { s.logger.Printf("warning: extension registration failed: %v", err) } else { diff --git a/server/grpc.go b/server/grpc.go index c27527af0..a0030d6f5 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -42,10 +42,16 @@ type grpcHandler struct { // to the error (returning it as a status.Error). It is // assumed that the input err is non-nil. func errToStatusError(err error) error { + // Check error string. switch errors.Cause(err) { case pilosa.ErrIndexNotFound, pilosa.ErrFieldNotFound: return status.Error(codes.NotFound, err.Error()) } + // Check error type. + switch errors.Cause(err).(type) { + case pilosa.NotFoundError: + return status.Error(codes.NotFound, err.Error()) + } return status.Error(codes.Unknown, err.Error()) } @@ -69,11 +75,36 @@ func (h grpcHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQL return nil } +// fieldDataType returns a useful data type (string, +// uint64, bool, etc.) based on the Pilosa field type. +func fieldDataType(f *pilosa.Field) string { + switch f.Type() { + case "set", "mutex": + if f.Options().Keys { + return "[]string" + } else { + return "[]uint64" + } + case "int": + return "int64" + case "decimal": + return "float64" + case "bool": + return "bool" + case "time": + return "int64" // TODO: this is a placeholder + default: + panic(fmt.Sprintf("unimplemented fieldDataType: %s", f.Type())) + } +} + // Inspect handles the inspect request and sends an InspectResponse to the stream. func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectServer) error { + const defaultLimit = 100000 + index, err := h.api.Index(context.Background(), req.Index) if err != nil { - return errors.Wrap(err, "getting index") + return errToStatusError(err) } var fields []*pilosa.Field @@ -91,20 +122,64 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer } } - // If there are no matching fields, then don't return any records. - if len(fields) == 0 { - return nil + limit := req.Limit + if limit == 0 { + limit = defaultLimit } + offset := req.Offset - if ints, ok := req.Columns.Type.(*pb.IdsOrKeys_Ids); ok { + if !index.Options().Keys { + ints, ok := req.Columns.Type.(*pb.IdsOrKeys_Ids) + if !ok { + return errors.New("invalid int columns") + } ci := []*pb.ColumnInfo{ {Name: "_id", Datatype: "uint64"}, } for _, field := range fields { - ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: field.Type()}) // TODO: field.Type likely doesn't align with supported datatypes + ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: fieldDataType(field)}) } - for _, col := range ints.Ids.Vals { + // If Columns is empty, then get the _exists list (via All()), + // from the index and loop over that instead. + cols := ints.Ids.Vals + if len(cols) > 0 { + // Apply limit/offset to the provided columns. + if int(offset) >= len(cols) { + return nil + } + end := limit + offset + if int(end) > len(cols) { + end = uint64(len(cols)) + } + cols = cols[offset:end] + } else { + // Prevent getting too many records by forcing a limit. + pql := fmt.Sprintf("All(limit=%d, offset=%d)", limit, offset) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + return errors.Wrapf(err, "querying for all: %s", pql) + } + + ids, ok := resp.Results[0].(*pilosa.Row) + if !ok { + return errors.Wrap(err, "getting results as a row") + } + + limitedCols := ids.Columns() + if len(limitedCols) == 0 { + // If cols is still empty after the limit/offset, then + // return with no results. + return nil + } + cols = limitedCols + } + + for _, col := range cols { rowResp := &pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{ @@ -178,6 +253,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + case "decimal": value, exists, err := field.FloatValue(col) if err != nil { @@ -189,6 +265,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + case "bool": pql := fmt.Sprintf("Rows(%s, column=%d)", field.Name(), col) query := pilosa.QueryRequest{ @@ -216,6 +293,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + case "time": rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) @@ -227,15 +305,58 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer } } - } else if keys, ok := req.Columns.Type.(*pb.IdsOrKeys_Keys); ok { + } else { + keys, ok := req.Columns.Type.(*pb.IdsOrKeys_Keys) + if !ok { + return errToStatusError(errors.New("invalid key columns")) + } ci := []*pb.ColumnInfo{ {Name: "_id", Datatype: "string"}, } for _, field := range fields { - ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: field.Type()}) // TODO: field.Type likely doesn't align with supported datatypes + ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: fieldDataType(field)}) } - for _, col := range keys.Keys.Vals { + // If Columns is empty, then get the _exists list (via All()), + // from the index and loop over that instead. + cols := keys.Keys.Vals + if len(cols) > 0 { + // Apply limit/offset to the provided columns. + if int(offset) >= len(cols) { + return nil + } + end := limit + offset + if int(end) > len(cols) { + end = uint64(len(cols)) + } + cols = cols[offset:end] + } else { + // Prevent getting too many records by forcing a limit. + pql := fmt.Sprintf("All(limit=%d, offset=%d)", limit, offset) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + return errors.Wrapf(err, "querying for all: %s", pql) + } + + ids, ok := resp.Results[0].(*pilosa.Row) + if !ok { + return errors.Wrap(err, "getting results as a row") + } + + limitedCols := ids.Keys + if len(limitedCols) == 0 { + // If cols is still empty after the limit/offset, then + // return with no results. + return nil + } + cols = limitedCols + } + + for _, col := range cols { rowResp := &pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{ @@ -433,35 +554,35 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo } */ } - case pilosa.Pair: - if r.Key != "" { + case pilosa.PairField: + if r.Pair.Key != "" { results <- &pb.RowResponse{ Headers: []*pb.ColumnInfo{ - {Name: "_id", Datatype: "string"}, + {Name: r.Field, Datatype: "string"}, {Name: "count", Datatype: "uint64"}, }, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: r.Key}}, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Count}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: r.Pair.Key}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.Count}}, }, } } else { results <- &pb.RowResponse{ Headers: []*pb.ColumnInfo{ - {Name: "_id", Datatype: "uint64"}, + {Name: r.Field, Datatype: "uint64"}, {Name: "count", Datatype: "uint64"}, }, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.ID}}, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Count}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.ID}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.Count}}, }, } } - case []pilosa.Pair: + case *pilosa.PairsField: // Determine if the ID has string keys. var stringKeys bool - if len(r) > 0 { - if r[0].Key != "" { + if len(r.Pairs) > 0 { + if r.Pairs[0].Key != "" { stringKeys = true } } @@ -471,10 +592,10 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo dtype = "string" } ci := []*pb.ColumnInfo{ - {Name: "_id", Datatype: dtype}, + {Name: r.Field, Datatype: dtype}, {Name: "count", Datatype: "uint64"}, } - for _, pair := range r { + for _, pair := range r.Pairs { if stringKeys { results <- &pb.RowResponse{ Headers: ci, @@ -528,7 +649,7 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo } case pilosa.RowIdentifiers: if len(r.Keys) > 0 { - ci := []*pb.ColumnInfo{{Name: "_id", Datatype: "string"}} + ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "string"}} for _, key := range r.Keys { results <- &pb.RowResponse{ Headers: ci, @@ -538,7 +659,7 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo ci = nil } } else { - ci := []*pb.ColumnInfo{{Name: "_id", Datatype: "uint64"}} + ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "uint64"}} for _, id := range r.Rows { results <- &pb.RowResponse{ Headers: ci, @@ -573,6 +694,27 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: r.Val}}, &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: r.Count}}, }} + case pilosa.SignedRow: + // TODO: address the overflow issue with values outside the int64 range + ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "int64"}} + negs := r.Neg.Columns() + for i := len(negs) - 1; i >= 0; i-- { + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: -1 * int64(negs[i])}}, + }} + ci = nil + } + for _, id := range r.Pos.Columns() { + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: int64(id)}}, + }} + ci = nil + } + default: logger.Printf("unhandled %T\n", r) breakLoop = true @@ -621,7 +763,7 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error { if err != nil { return errors.Wrap(err, "creating listener") } - s.logger.Printf("enabled grpc listening on %s", s.hostPort) + s.logger.Printf("enabled grpc listening on %s", lis.Addr()) opts := make([]grpc.ServerOption, 0) if tlsConfig != nil { diff --git a/server/grpc_internal_test.go b/server/grpc_internal_test.go index 285f35eb1..c0701be3e 100644 --- a/server/grpc_internal_test.go +++ b/server/grpc_internal_test.go @@ -175,7 +175,7 @@ func TestGRPC(t *testing.T) { Rows: []uint64{10, 11, 12}, }, []expHeader{ - {"_id", "uint64"}, + {"", "uint64"}, // This is blank because we don't expose RowIdentifiers.field, so we have no way to set it for tests. }, [][]expColumn{ {uint64(10)}, @@ -189,7 +189,7 @@ func TestGRPC(t *testing.T) { Keys: []string{"ten", "eleven", "twelve"}, }, []expHeader{ - {"_id", "string"}, + {"", "string"}, // This is blank because we don't expose RowIdentifiers.field, so we have no way to set it for tests. }, [][]expColumn{ {"ten"}, diff --git a/server/handler_test.go b/server/handler_test.go index c67d98657..2f0af0345 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -224,7 +224,12 @@ func TestHandler_Endpoints(t *testing.T) { if err != nil { t.Fatalf("querying: %v", err) } - if !reflect.DeepEqual(resp.Results[0], []pilosa.Pair{{Count: 12, ID: 0}}) { + if !reflect.DeepEqual(resp.Results[0], &pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {Count: 12, ID: 0}, + }, + Field: "f1", + }) { t.Fatalf("Unexpected result %v", resp.Results[0]) } @@ -504,8 +509,8 @@ func TestHandler_Endpoints(t *testing.T) { var resp pilosa.QueryResponse if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if a := resp.Results[0].([]pilosa.Pair); len(a) != 2 { - t.Fatalf("unexpected pair length: %d", len(a)) + } else if a := resp.Results[0].(*pilosa.PairsField); len(a.Pairs) != 2 { + t.Fatalf("unexpected pair length: %d", len(a.Pairs)) } }) diff --git a/stats/stats.go b/stats/stats.go index 9c6944d4e..c360baab0 100644 --- a/stats/stats.go +++ b/stats/stats.go @@ -25,7 +25,7 @@ import ( ) // Expvar global expvar map. -var Expvar = expvar.NewMap("index") +var Expvar *expvar.Map // StatsClient represents a client to a stats server. type StatsClient interface { @@ -90,6 +90,9 @@ type expvarStatsClient struct { // NewExpvarStatsClient returns a new instance of ExpvarStatsClient. // This client points at the root of the expvar index map. func NewExpvarStatsClient() *expvarStatsClient { + if Expvar == nil { + Expvar = expvar.NewMap("index") + } return &expvarStatsClient{ m: Expvar, }